diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index cf4ba5aa114..83078a304ed 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,12 +5,12 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index e86a38abbde..d59742575b5 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,13 +5,15 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-llvm16-cuda11.8-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/ucx:23.12": { + "version": "1.14.1" + }, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json index 863eeea48ff..bc4a2cb6fb4 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -5,12 +5,12 @@ "args": { "CUDA": "12.0", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:23.10-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-mambaforge-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index c7612771fd3..3eacf726bf0 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -5,13 +5,15 @@ "args": { "CUDA": "12.0", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda12.0-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-llvm16-cuda12.0-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/ucx:23.12": { + "version": "1.14.1" + }, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c01a6fcb94a..0f490283795 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,13 +22,13 @@ on: default: nightly concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-23.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -77,12 +77,13 @@ jobs: date: ${{ inputs.date }} script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.10 + extra-repo-sha: branch-23.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + node_type: cpu32 wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -92,7 +93,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,12 +101,12 @@ jobs: date: ${{ inputs.date }} script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.10 + extra-repo-sha: branch-23.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -115,7 +116,7 @@ jobs: wheel-build-nx-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -125,7 +126,7 @@ jobs: wheel-publish-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/github/workflows/labeler.yml b/.github/workflows/labeler.yml similarity index 83% rename from github/workflows/labeler.yml rename to .github/workflows/labeler.yml index 23956a02fbd..31e78f82a62 100644 --- a/github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,6 +6,6 @@ jobs: triage: runs-on: ubuntu-latest steps: - - uses: actions/labeler@main + - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7b267d7edf3..9d20074381e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,41 +27,41 @@ jobs: - wheel-tests-nx-cugraph - devcontainer secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-23.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.12 with: build_type: pull-request node_type: cpu32 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.12 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-23.12 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-23.12 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-23.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -71,7 +71,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-23.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -81,57 +81,59 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.10 + extra-repo-sha: branch-23.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + node_type: cpu32 wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: pull-request script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.10 + extra-repo-sha: branch-23.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: pull-request script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh devcontainer: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-23.12 with: + node_type: cpu32 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY build_command: | sccache -z; - build-all --verbose; + build-all --verbose -j$(nproc --ignore=1); sccache -s; diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dc9ed60b29e..a0ecb67712c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-23.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -41,7 +41,7 @@ jobs: script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -50,7 +50,7 @@ jobs: script: ci/test_wheel_cugraph.sh wheel-tests-nx-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/.gitignore b/.gitignore index c6bcf6965d7..358650cfc5a 100644 --- a/.gitignore +++ b/.gitignore @@ -84,8 +84,10 @@ datasets/* # Jupyter Notebooks .ipynb_checkpoints -## Doxygen +## Doxygen and Docs cpp/doxygen/html +docs/cugraph/lib* +docs/cugraph/api/* # created by Dask tests python/dask-worker-space diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 865d06b20e4..bab39557c99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - id: black language_version: python3 args: [--target-version=py38] - files: ^python/ + files: ^(python/.*|benchmarks/.*)$ - repo: https://github.com/PyCQA/flake8 rev: 6.0.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9181b8b0ea2..37c17c0af1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,84 @@ +# cuGraph 23.12.00 (6 Dec 2023) + +## 🚨 Breaking Changes + +- [BUG] Restore the original default order of CSR, which does not reverse edges in cuGraph-PyG ([#3980](https://github.com/rapidsai/cugraph/pull/3980)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- `Resultset` and `Dataset` Refactors ([#3957](https://github.com/rapidsai/cugraph/pull/3957)) [@nv-rliu](https://github.com/nv-rliu) +- Moves more MG graph ETL to libcugraph and re-enables MG tests in CI ([#3941](https://github.com/rapidsai/cugraph/pull/3941)) [@jnke2016](https://github.com/jnke2016) + +## 🐛 Bug Fixes + +- Pin actions/labeler to v4 ([#4038](https://github.com/rapidsai/cugraph/pull/4038)) [@raydouglass](https://github.com/raydouglass) +- Find rmm before cuco ([#4011](https://github.com/rapidsai/cugraph/pull/4011)) [@vyasr](https://github.com/vyasr) +- Pin to minor versions of packages outside the cuGraph repository. ([#4004](https://github.com/rapidsai/cugraph/pull/4004)) [@bdice](https://github.com/bdice) +- Move MTMG_TEST to MG tests block ([#3993](https://github.com/rapidsai/cugraph/pull/3993)) [@naimnv](https://github.com/naimnv) +- Fix Leiden refinement phase ([#3990](https://github.com/rapidsai/cugraph/pull/3990)) [@naimnv](https://github.com/naimnv) +- [BUG] Fix Graph Construction From Pandas in cuGraph-PyG ([#3985](https://github.com/rapidsai/cugraph/pull/3985)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- [BUG] Restore the original default order of CSR, which does not reverse edges in cuGraph-PyG ([#3980](https://github.com/rapidsai/cugraph/pull/3980)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- Fix eigenvector testing and HITS testing discrepancies ([#3979](https://github.com/rapidsai/cugraph/pull/3979)) [@ChuckHastings](https://github.com/ChuckHastings) +- [BUG] Fix Incorrect Edge Index, Directory Selection in cuGraph-PyG Loader ([#3978](https://github.com/rapidsai/cugraph/pull/3978)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- [BUG] Check if Dask has quit to avoid throwing an exception and triggering a segfault on ddp exit ([#3961](https://github.com/rapidsai/cugraph/pull/3961)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- nx-cugraph: xfail test_louvain.py:test_threshold in Python 3.9 ([#3944](https://github.com/rapidsai/cugraph/pull/3944)) [@eriknw](https://github.com/eriknw) + +## 📖 Documentation + +- [DOC]: Fix invalid links and add materials to notebooks ([#4002](https://github.com/rapidsai/cugraph/pull/4002)) [@huiyuxie](https://github.com/huiyuxie) +- Update Broken Links in README.md ([#3924](https://github.com/rapidsai/cugraph/pull/3924)) [@nv-rliu](https://github.com/nv-rliu) + +## 🚀 New Features + +- Implement the transform_e primitive (to update property values for all edges) ([#3917](https://github.com/rapidsai/cugraph/pull/3917)) [@seunghwak](https://github.com/seunghwak) +- Update the neighbor intersection primitive to support edge masking. ([#3550](https://github.com/rapidsai/cugraph/pull/3550)) [@seunghwak](https://github.com/seunghwak) + +## 🛠️ Improvements + +- Correct defect found in DLFW testing ([#4021](https://github.com/rapidsai/cugraph/pull/4021)) [@ChuckHastings](https://github.com/ChuckHastings) +- `nx-cugraph` README update: adds missing `connected_components` algo to table ([#4019](https://github.com/rapidsai/cugraph/pull/4019)) [@rlratzel](https://github.com/rlratzel) +- Build concurrency for nightly and merge triggers ([#4009](https://github.com/rapidsai/cugraph/pull/4009)) [@bdice](https://github.com/bdice) +- Support `drop_last` Argument in cuGraph-PyG Loader ([#3995](https://github.com/rapidsai/cugraph/pull/3995)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- Adds `update-version.sh` support for recently added files containing RAPIDS versions ([#3994](https://github.com/rapidsai/cugraph/pull/3994)) [@rlratzel](https://github.com/rlratzel) +- Use new `rapids-dask-dependency` metapackage for managing `dask` versions ([#3991](https://github.com/rapidsai/cugraph/pull/3991)) [@galipremsagar](https://github.com/galipremsagar) +- Fixes to nx-cugraph README: fixes typos, updates link to NX backend docs ([#3989](https://github.com/rapidsai/cugraph/pull/3989)) [@rlratzel](https://github.com/rlratzel) +- Address FIXMEs ([#3988](https://github.com/rapidsai/cugraph/pull/3988)) [@seunghwak](https://github.com/seunghwak) +- Updates README file to include nx-cugraph user documentation, adds nx-cugraph to main README ([#3984](https://github.com/rapidsai/cugraph/pull/3984)) [@rlratzel](https://github.com/rlratzel) +- Update C API graph creation function signatures ([#3982](https://github.com/rapidsai/cugraph/pull/3982)) [@ChuckHastings](https://github.com/ChuckHastings) +- [REVIEW]Optimize cugraph-DGL csc codepath ([#3977](https://github.com/rapidsai/cugraph/pull/3977)) [@VibhuJawa](https://github.com/VibhuJawa) +- nx-cugraph: add SSSP (unweighted) ([#3976](https://github.com/rapidsai/cugraph/pull/3976)) [@eriknw](https://github.com/eriknw) +- CuGraph compatibility fixes ([#3973](https://github.com/rapidsai/cugraph/pull/3973)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Skip certain `cugraph-pyg` tests when torch-sparse is not available ([#3970](https://github.com/rapidsai/cugraph/pull/3970)) [@tingyu66](https://github.com/tingyu66) +- nx-cugraph: add `eigenvector_centrality`, `katz_centrality`, `hits`, `pagerank` ([#3968](https://github.com/rapidsai/cugraph/pull/3968)) [@eriknw](https://github.com/eriknw) +- Cut peak memory footprint in graph creation ([#3966](https://github.com/rapidsai/cugraph/pull/3966)) [@seunghwak](https://github.com/seunghwak) +- nx-cugraph: add CC for undirected graphs to fix k-truss ([#3965](https://github.com/rapidsai/cugraph/pull/3965)) [@eriknw](https://github.com/eriknw) +- Skip certain `cugraph-pyg` tests when `torch_sparse` is not available ([#3962](https://github.com/rapidsai/cugraph/pull/3962)) [@tingyu66](https://github.com/tingyu66) +- `Resultset` and `Dataset` Refactors ([#3957](https://github.com/rapidsai/cugraph/pull/3957)) [@nv-rliu](https://github.com/nv-rliu) +- Download `xml` docs artifact through CloudFront endpoint ([#3955](https://github.com/rapidsai/cugraph/pull/3955)) [@AyodeAwe](https://github.com/AyodeAwe) +- Add many graph generators to nx-cugraph ([#3954](https://github.com/rapidsai/cugraph/pull/3954)) [@eriknw](https://github.com/eriknw) +- Unpin `dask` and `distributed` for `23.12` development ([#3953](https://github.com/rapidsai/cugraph/pull/3953)) [@galipremsagar](https://github.com/galipremsagar) +- Errors compiling for DLFW on CUDA 12.3 ([#3952](https://github.com/rapidsai/cugraph/pull/3952)) [@ChuckHastings](https://github.com/ChuckHastings) +- nx-cugraph: add k_truss and degree centralities ([#3945](https://github.com/rapidsai/cugraph/pull/3945)) [@eriknw](https://github.com/eriknw) +- nx-cugraph: handle seed argument in edge_betweenness_centrality ([#3943](https://github.com/rapidsai/cugraph/pull/3943)) [@eriknw](https://github.com/eriknw) +- Moves more MG graph ETL to libcugraph and re-enables MG tests in CI ([#3941](https://github.com/rapidsai/cugraph/pull/3941)) [@jnke2016](https://github.com/jnke2016) +- Temporarily disable mg testing ([#3940](https://github.com/rapidsai/cugraph/pull/3940)) [@jnke2016](https://github.com/jnke2016) +- adding C/C++ API docs ([#3938](https://github.com/rapidsai/cugraph/pull/3938)) [@BradReesWork](https://github.com/BradReesWork) +- Add multigraph support to nx-cugraph ([#3934](https://github.com/rapidsai/cugraph/pull/3934)) [@eriknw](https://github.com/eriknw) +- Setup Consistent Nightly Versions for Pip and Conda ([#3933](https://github.com/rapidsai/cugraph/pull/3933)) [@divyegala](https://github.com/divyegala) +- MTMG multi node ([#3932](https://github.com/rapidsai/cugraph/pull/3932)) [@ChuckHastings](https://github.com/ChuckHastings) +- Use branch-23.12 workflows. ([#3928](https://github.com/rapidsai/cugraph/pull/3928)) [@bdice](https://github.com/bdice) +- Fix an issue occurring in the cuGraph-DGL example for "mixed" mode. ([#3927](https://github.com/rapidsai/cugraph/pull/3927)) [@drivanov](https://github.com/drivanov) +- Updating Docs ([#3923](https://github.com/rapidsai/cugraph/pull/3923)) [@BradReesWork](https://github.com/BradReesWork) +- Forward-merge branch-23.10 to branch-23.12 ([#3919](https://github.com/rapidsai/cugraph/pull/3919)) [@nv-rliu](https://github.com/nv-rliu) +- new build all option ([#3916](https://github.com/rapidsai/cugraph/pull/3916)) [@BradReesWork](https://github.com/BradReesWork) +- Silence spurious compiler warnings ([#3913](https://github.com/rapidsai/cugraph/pull/3913)) [@seunghwak](https://github.com/seunghwak) +- Link wholegrah and cugraphops XML docs ([#3906](https://github.com/rapidsai/cugraph/pull/3906)) [@AyodeAwe](https://github.com/AyodeAwe) +- Updates to 23.12 ([#3905](https://github.com/rapidsai/cugraph/pull/3905)) [@raydouglass](https://github.com/raydouglass) +- Forward-merge branch-23.10 to branch-23.12 ([#3904](https://github.com/rapidsai/cugraph/pull/3904)) [@GPUtester](https://github.com/GPUtester) +- Build CUDA 12.0 ARM conda packages. ([#3903](https://github.com/rapidsai/cugraph/pull/3903)) [@bdice](https://github.com/bdice) +- Merge branch-23.10 into branch-23.12 ([#3898](https://github.com/rapidsai/cugraph/pull/3898)) [@rlratzel](https://github.com/rlratzel) +- Some MTMG code cleanup and small optimizations ([#3894](https://github.com/rapidsai/cugraph/pull/3894)) [@ChuckHastings](https://github.com/ChuckHastings) +- Enable parallel mode ([#3875](https://github.com/rapidsai/cugraph/pull/3875)) [@jnke2016](https://github.com/jnke2016) +- Adds benchmarks for `nx-cugraph` ([#3854](https://github.com/rapidsai/cugraph/pull/3854)) [@rlratzel](https://github.com/rlratzel) +- Add nx-cugraph notebook for showing accelerated networkX APIs ([#3830](https://github.com/rapidsai/cugraph/pull/3830)) [@betochimas](https://github.com/betochimas) + # cuGraph 23.10.00 (11 Oct 2023) ## 🚨 Breaking Changes diff --git a/README.md b/README.md index 3daeb0570b0..8026e4feb64 100644 --- a/README.md +++ b/README.md @@ -35,16 +35,26 @@ ----- +## News -## Table of content +___NEW!___ _[nx-cugraph](./python/nx-cugraph/README.md)_, a NetworkX backend that provides GPU acceleration to NetworkX with zero code change. +``` +> pip install nx-cugraph-cu11 --extra-index-url https://pypi.nvidia.com +> export NETWORKX_AUTOMATIC_BACKENDS=cugraph +``` +That's it. NetworkX now leverages cuGraph for accelerated graph algorithms. + +----- + +## Table of contents - Installation - [Getting cuGraph Packages](./docs/cugraph/source/installation/getting_cugraph.md) - [Building from Source](./docs/cugraph/source/installation/source_build.md) - [Contributing to cuGraph](./readme_pages/CONTRIBUTING.md) - General - [Latest News](./readme_pages/news.md) - - [Current list of algorithms](./readme_pages/algorithms.md) - - [Blogs and Presentation](./docs/cugraph/source/basics/cugraph_blogs.rst) + - [Current list of algorithms](./docs/cugraph/source/graph_support/algorithms.md) + - [Blogs and Presentation](./docs/cugraph/source/tutorials/cugraph_blogs.rst) - [Performance](./readme_pages/performance/performance.md) - Packages - [cuGraph Python](./readme_pages/cugraph_python.md) @@ -52,6 +62,7 @@ - [External Data Types](./readme_pages/data_types.md) - [pylibcugraph](./readme_pages/pylibcugraph.md) - [libcugraph (C/C++/CUDA)](./readme_pages/libcugraph.md) + - [nx-cugraph](./python/nx-cugraph/README.md) - [cugraph-service](./readme_pages/cugraph_service.md) - [cugraph-dgl](./readme_pages/cugraph_dgl.md) - [cugraph-ops](./readme_pages/cugraph_ops.md) @@ -116,6 +127,7 @@ df_page.sort_values('pagerank', ascending=False).head(10) * ArangoDB - a free and open-source native multi-model database system - https://www.arangodb.com/ * CuPy - "NumPy/SciPy-compatible Array Library for GPU-accelerated Computing with Python" - https://cupy.dev/ * Memgraph - In-memory Graph database - https://memgraph.com/ +* NetworkX (via [nx-cugraph](./python/nx-cugraph/README.md) backend) - an extremely popular, free and open-source package for the creation, manipulation, and study of the structure, dynamics, and functions of complex networks - https://networkx.org/ * PyGraphistry - free and open-source GPU graph ETL, AI, and visualization, including native RAPIDS & cuGraph support - http://github.com/graphistry/pygraphistry * ScanPy - a scalable toolkit for analyzing single-cell gene expression data - https://scanpy.readthedocs.io/en/stable/ diff --git a/VERSION b/VERSION new file mode 100644 index 00000000000..a193fff41e8 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +23.12.00 diff --git a/benchmarks/cugraph-dgl/pytest-based/bench_cugraph_dgl_uniform_neighbor_sample.py b/benchmarks/cugraph-dgl/pytest-based/bench_cugraph_dgl_uniform_neighbor_sample.py index eeee163b0af..dc961223cb1 100644 --- a/benchmarks/cugraph-dgl/pytest-based/bench_cugraph_dgl_uniform_neighbor_sample.py +++ b/benchmarks/cugraph-dgl/pytest-based/bench_cugraph_dgl_uniform_neighbor_sample.py @@ -21,6 +21,7 @@ import pytest import numpy as np import cupy as cp + # Facing issues with rapids-pytest-benchmark plugin # pytest-benchmark. import pytest_benchmark @@ -33,6 +34,7 @@ import dgl import torch import rmm + _seed = 42 @@ -71,21 +73,21 @@ def create_graph(graph_data): else: raise TypeError(f"graph_data can only be str or dict, got {type(graph_data)}") - num_nodes = max(edgelist_df['src'].max(), - edgelist_df['dst'].max())+1 + num_nodes = max(edgelist_df["src"].max(), edgelist_df["dst"].max()) + 1 - num_nodes_dict = {'_N':num_nodes} + num_nodes_dict = {"_N": num_nodes} gs = CuGraphStorage(num_nodes_dict=num_nodes_dict, single_gpu=True) - gs.add_edge_data(edgelist_df, - # reverse to make same graph as cugraph - node_col_names=['dst', 'src'], - canonical_etype=['_N', 'connects', '_N']) + gs.add_edge_data( + edgelist_df, + # reverse to make same graph as cugraph + node_col_names=["dst", "src"], + canonical_etype=["_N", "connects", "_N"], + ) return gs - def create_mg_graph(graph_data): """ Create a graph instance based on the data to be loaded/generated. @@ -93,7 +95,9 @@ def create_mg_graph(graph_data): # range starts at 1 to let let 0 be used by benchmark/client process visible_devices = os.getenv("DASK_WORKER_DEVICES", "1,2,3,4") - cluster = LocalCUDACluster(protocol='ucx', rmm_pool_size='25GB', CUDA_VISIBLE_DEVICES=visible_devices) + cluster = LocalCUDACluster( + protocol="ucx", rmm_pool_size="25GB", CUDA_VISIBLE_DEVICES=visible_devices + ) client = Client(cluster) Comms.initialize(p2p=True) rmm.reinitialize(pool_allocator=True) @@ -126,25 +130,23 @@ def create_mg_graph(graph_data): else: raise TypeError(f"graph_data can only be str or dict, got {type(graph_data)}") - num_nodes = max(edgelist_df['src'].max().compute(), - edgelist_df['dst'].max().compute()) + num_nodes = max( + edgelist_df["src"].max().compute(), edgelist_df["dst"].max().compute() + ) # running into issues with smaller partitions - edgelist_df = edgelist_df.repartition(npartitions=edgelist_df.npartitions*2) + edgelist_df = edgelist_df.repartition(npartitions=edgelist_df.npartitions * 2) - num_nodes_dict = {'_N':num_nodes} + num_nodes_dict = {"_N": num_nodes} - gs = CuGraphStorage(num_nodes_dict=num_nodes_dict, single_gpu=False) - gs.add_edge_data(edgelist_df, - node_col_names=['dst', 'src'], - canonical_etype=['_N', 'C', '_N']) + gs = CuGraphStorage(num_nodes_dict=num_nodes_dict, single_gpu=False) + gs.add_edge_data( + edgelist_df, node_col_names=["dst", "src"], canonical_etype=["_N", "C", "_N"] + ) return (gs, client, cluster) - -def get_uniform_neighbor_sample_args( - G, seed, batch_size, fanout, with_replacement -): +def get_uniform_neighbor_sample_args(G, seed, batch_size, fanout, with_replacement): """ Return a dictionary containing the args for uniform_neighbor_sample based on the graph and desired args passed in. For example, if a large start list @@ -165,7 +167,7 @@ def get_uniform_neighbor_sample_args( else: num_start_verts = batch_size - srcs = G.graphstore.gdata.get_edge_data()['_SRC_'] + srcs = G.graphstore.gdata.get_edge_data()["_SRC_"] start_list = srcs.head(num_start_verts) assert len(start_list) == num_start_verts @@ -205,7 +207,6 @@ def graph_objs(request): dask_cluster.close() - ################################################################################ # Benchmarks @pytest.mark.parametrize("batch_size", params.batch_sizes.values()) @@ -223,7 +224,7 @@ def bench_cugraph_dgl_uniform_neighbor_sample( # Reverse to match cugraph # DGL does from dst to src - fanout_val = uns_args['fanout'] + fanout_val = uns_args["fanout"] fanout_val.reverse() sampler = dgl.dataloading.NeighborSampler(uns_args["fanout"]) sampler_f = sampler.sample_blocks @@ -231,7 +232,7 @@ def bench_cugraph_dgl_uniform_neighbor_sample( # Warmup _ = sampler_f(g=G, seed_nodes=uns_args["seed_nodes"]) # print(f"\n{uns_args}") - result_seed_nodes, output_nodes, blocks = benchmark( + result_seed_nodes, output_nodes, blocks = benchmark( sampler_f, g=G, seed_nodes=uns_args["seed_nodes"], diff --git a/benchmarks/cugraph-dgl/python-script/dgl_dataloading_benchmark/dgl_benchmark.py b/benchmarks/cugraph-dgl/python-script/dgl_dataloading_benchmark/dgl_benchmark.py index 0a52703c546..fc7543a91e0 100644 --- a/benchmarks/cugraph-dgl/python-script/dgl_dataloading_benchmark/dgl_benchmark.py +++ b/benchmarks/cugraph-dgl/python-script/dgl_dataloading_benchmark/dgl_benchmark.py @@ -52,13 +52,9 @@ def load_edges_from_disk(parquet_path, replication_factor, input_meta): src_ls = [ei["src"]] dst_ls = [ei["dst"]] for r in range(1, replication_factor): - new_src = ei["src"] + ( - r * input_meta["num_nodes"][can_edge_type[0]] - ) + new_src = ei["src"] + (r * input_meta["num_nodes"][can_edge_type[0]]) src_ls.append(new_src) - new_dst = ei["dst"] + ( - r * input_meta["num_nodes"][can_edge_type[2]] - ) + new_dst = ei["dst"] + (r * input_meta["num_nodes"][can_edge_type[2]]) dst_ls.append(new_dst) ei["src"] = torch.cat(src_ls).contiguous() @@ -92,16 +88,11 @@ def load_node_labels(dataset_path, replication_factor, input_meta): ] ), "label": pd.concat( - [ - node_label.label - for r in range(1, replication_factor) - ] + [node_label.label for r in range(1, replication_factor)] ), } ) - node_label = pd.concat([node_label, dfr]).reset_index( - drop=True - ) + node_label = pd.concat([node_label, dfr]).reset_index(drop=True) node_label_tensor = torch.full( (num_nodes_dict[node_type],), -1, dtype=torch.float32 @@ -133,9 +124,7 @@ def create_dgl_graph_from_disk(dataset_path, replication_factor=1): input_meta = json.load(f) parquet_path = os.path.join(dataset_path, "parquet") - graph_data = load_edges_from_disk( - parquet_path, replication_factor, input_meta - ) + graph_data = load_edges_from_disk(parquet_path, replication_factor, input_meta) node_data = load_node_labels(dataset_path, replication_factor, input_meta) g = dgl.heterograph(graph_data) @@ -154,7 +143,7 @@ def create_dataloader(g, train_idx, batch_size, fanouts, use_uva): Returns: DGLGraph: DGLGraph with the loaded dataset. """ - + print("Creating dataloader", flush=True) st = time.time() if use_uva: @@ -220,21 +209,21 @@ def dataloading_benchmark(g, train_idx, fanouts, batch_sizes, use_uva): print("==============================================") return time_ls + def set_seed(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) + if __name__ == "__main__": parser = ArgumentParser() parser.add_argument( "--dataset_path", type=str, default="/datasets/abarghi/ogbn_papers100M" ) parser.add_argument("--replication_factors", type=str, default="1,2,4,8") - parser.add_argument( - "--fanouts", type=str, default="25_25,10_10_10,5_10_20" - ) + parser.add_argument("--fanouts", type=str, default="25_25,10_10_10,5_10_20") parser.add_argument("--batch_sizes", type=str, default="512,1024") parser.add_argument("--do_not_use_uva", action="store_true") parser.add_argument("--seed", type=int, default=42) @@ -267,22 +256,16 @@ def set_seed(seed): et = time.time() print(f"Replication factor = {replication_factor}") print( - f"G has {g.num_edges()} edges and took", - f" {et - st:.2f} seconds to load" + f"G has {g.num_edges()} edges and took", f" {et - st:.2f} seconds to load" ) train_idx = {"paper": node_data["paper"]["train_idx"]} r_time_ls = dataloading_benchmark( g, train_idx, fanouts, batch_sizes, use_uva=use_uva ) - print( - "Benchmark completed for replication factor = ", replication_factor - ) + print("Benchmark completed for replication factor = ", replication_factor) print("==============================================") # Add replication factor to the time list - [ - x.update({"replication_factor": replication_factor}) - for x in r_time_ls - ] + [x.update({"replication_factor": replication_factor}) for x in r_time_ls] time_ls.extend(r_time_ls) df = pd.DataFrame(time_ls) diff --git a/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py b/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py index 7c8b352d7af..539fe333b1e 100644 --- a/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py +++ b/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py @@ -16,15 +16,17 @@ import torch.nn.functional as F import time import argparse + ## DGL Specific Import from cugraph_dgl import c + def load_dgl_graph(): from ogb.nodeproppred import DglNodePropPredDataset - dataset = DglNodePropPredDataset(name="ogbn-mag", root='/datasets/vjawa/gnn/') + dataset = DglNodePropPredDataset(name="ogbn-mag", root="/datasets/vjawa/gnn/") split_idx = dataset.get_idx_split() - g, labels = dataset[0] + g, labels = dataset[0] # Uncomment for obgn-mag labels = labels["paper"].flatten() # labels = labels @@ -33,26 +35,30 @@ def load_dgl_graph(): return g, labels, dataset.num_classes, split_idx -def sampling_func(g, seed_nodes,labels, train_loader): +def sampling_func(g, seed_nodes, labels, train_loader): category = "paper" for input_nodes, seeds, blocks in train_loader: seeds = seeds[category] - feat = blocks[0].srcdata['feat']['paper'] + feat = blocks[0].srcdata["feat"]["paper"] return None if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Benchmark Sampling') - parser.add_argument('--batch_size', type=int, default=100_000) - parser.add_argument('--use_cugraph',dest='use_cugraph', action='store_true', default=True) - parser.add_argument('--use_dgl_upstream', dest='use_cugraph', action='store_false') - parser.add_argument('--single_gpu', dest='single_gpu', action='store_true', default=True) - parser.add_argument('--multi_gpu', dest='single_gpu', action='store_false') - parser.add_argument('--n_gpus', type=int, default=1) - + parser = argparse.ArgumentParser(description="Benchmark Sampling") + parser.add_argument("--batch_size", type=int, default=100_000) + parser.add_argument( + "--use_cugraph", dest="use_cugraph", action="store_true", default=True + ) + parser.add_argument("--use_dgl_upstream", dest="use_cugraph", action="store_false") + parser.add_argument( + "--single_gpu", dest="single_gpu", action="store_true", default=True + ) + parser.add_argument("--multi_gpu", dest="single_gpu", action="store_false") + parser.add_argument("--n_gpus", type=int, default=1) + args = parser.parse_args() print(args, flush=True) - + single_gpu = args.single_gpu use_cugraph = args.use_cugraph batch_size = args.batch_size @@ -60,42 +66,49 @@ def sampling_func(g, seed_nodes,labels, train_loader): if single_gpu: import rmm - rmm.reinitialize(pool_allocator=True,initial_pool_size=5e+9, maximum_pool_size=22e+9) + + rmm.reinitialize( + pool_allocator=True, initial_pool_size=5e9, maximum_pool_size=22e9 + ) else: #### Dask Cluster from dask_cuda import LocalCUDACluster from cugraph.dask.comms import comms as Comms from dask.distributed import Client - #Change according to your GPUS - #Client at GPU-0 - #Workers at specifed GPUS - #UCX seems to be freezing :-0 on DGX - cuda_visible_devices = ','.join([str(i) for i in range(1,n_gpus+1)]) - cluster = LocalCUDACluster(CUDA_VISIBLE_DEVICES=cuda_visible_devices, protocol='tcp', rmm_pool_size='12 GB', - jit_unspill=True) + # Change according to your GPUS + # Client at GPU-0 + # Workers at specifed GPUS + # UCX seems to be freezing :-0 on DGX + cuda_visible_devices = ",".join([str(i) for i in range(1, n_gpus + 1)]) + cluster = LocalCUDACluster( + CUDA_VISIBLE_DEVICES=cuda_visible_devices, + protocol="tcp", + rmm_pool_size="12 GB", + jit_unspill=True, + ) client = Client(cluster) Comms.initialize(p2p=True) - - - device = 'cuda' + + device = "cuda" g, labels, num_classes, split_idx = load_dgl_graph() g = g.to(device) - + if use_cugraph: if not single_gpu: g = g.int() g = cugraph_storage_from_heterograph(g, single_gpu=single_gpu) - - - indx_type = g.idtype - subset_split_idx = {'train': {k: v.to(device).to(g.idtype) for k,v in split_idx['train'].items()}, - 'valid' : {k: v.to(device).to(g.idtype) for k,v in split_idx['valid'].items()}, - 'test' : {k: v.to(device).to(g.idtype) for k,v in split_idx['test'].items()}, - } + indx_type = g.idtype + subset_split_idx = { + "train": {k: v.to(device).to(g.idtype) for k, v in split_idx["train"].items()}, + "valid": {k: v.to(device).to(g.idtype) for k, v in split_idx["valid"].items()}, + "test": {k: v.to(device).to(g.idtype) for k, v in split_idx["test"].items()}, + } - sampler = dgl.dataloading.MultiLayerNeighborSampler([20,25], prefetch_node_feats={'paper':['feat']}) + sampler = dgl.dataloading.MultiLayerNeighborSampler( + [20, 25], prefetch_node_feats={"paper": ["feat"]} + ) train_loader = dgl.dataloading.DataLoader( g, subset_split_idx["train"], @@ -107,10 +120,10 @@ def sampling_func(g, seed_nodes,labels, train_loader): ) ### Warmup RUN - sampling_func(g, subset_split_idx['train'],labels, train_loader) - + sampling_func(g, subset_split_idx["train"], labels, train_loader) + ### Benchmarking RUN st = time.time() - sampling_func(g, subset_split_idx['train'],labels, train_loader) - et = time.time() - print(f"Sampling time taken = {et-st} s") \ No newline at end of file + sampling_func(g, subset_split_idx["train"], labels, train_loader) + et = time.time() + print(f"Sampling time taken = {et-st} s") diff --git a/benchmarks/cugraph-dgl/scale-benchmarks/cugraph_dgl_benchmark.py b/benchmarks/cugraph-dgl/scale-benchmarks/cugraph_dgl_benchmark.py new file mode 100644 index 00000000000..85f43b97b90 --- /dev/null +++ b/benchmarks/cugraph-dgl/scale-benchmarks/cugraph_dgl_benchmark.py @@ -0,0 +1,152 @@ +# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +os.environ["LIBCUDF_CUFILE_POLICY"] = "KVIKIO" +os.environ["KVIKIO_NTHREADS"] = "64" +os.environ["RAPIDS_NO_INITIALIZE"] = "1" +import json +import pandas as pd +import os +import time +from rmm.allocators.torch import rmm_torch_allocator +import rmm +import torch +from cugraph_dgl.dataloading import HomogenousBulkSamplerDataset +from model import run_1_epoch +from argparse import ArgumentParser +from load_graph_feats import load_node_labels, load_node_features + + +def create_dataloader(sampled_dir, total_num_nodes, sparse_format, return_type): + print("Creating dataloader", flush=True) + st = time.time() + dataset = HomogenousBulkSamplerDataset( + total_num_nodes, + edge_dir="in", + sparse_format=sparse_format, + return_type=return_type, + ) + + dataset.set_input_files(sampled_dir) + dataloader = torch.utils.data.DataLoader( + dataset, collate_fn=lambda x: x, shuffle=False, num_workers=0, batch_size=None + ) + et = time.time() + print(f"Time to create dataloader = {et - st:.2f} seconds", flush=True) + return dataloader + + +def setup_common_pool(): + rmm.reinitialize(initial_pool_size=5e9, pool_allocator=True) + torch.cuda.memory.change_current_allocator(rmm_torch_allocator) + + +def main(args): + print( + f"Running cugraph-dgl dataloading benchmark with the following parameters:\n" + f"Dataset path = {args.dataset_path}\n" + f"Sampling path = {args.sampling_path}\n" + ) + with open(os.path.join(args.dataset_path, "meta.json"), "r") as f: + input_meta = json.load(f) + + sampled_dirs = [ + os.path.join(args.sampling_path, f) for f in os.listdir(args.sampling_path) + ] + + time_ls = [] + for sampled_dir in sampled_dirs: + with open(os.path.join(sampled_dir, "output_meta.json"), "r") as f: + sampled_meta_d = json.load(f) + + replication_factor = sampled_meta_d["replication_factor"] + feat_load_st = time.time() + label_data = load_node_labels( + args.dataset_path, replication_factor, input_meta + )["paper"]["y"] + feat_data = feat_data = load_node_features( + args.dataset_path, replication_factor, node_type="paper" + ) + print( + f"Feature and label data loading took = {time.time()-feat_load_st}", + flush=True, + ) + + r_time_ls = e2e_benchmark(sampled_dir, feat_data, label_data, sampled_meta_d) + [x.update({"replication_factor": replication_factor}) for x in r_time_ls] + [x.update({"num_edges": sampled_meta_d["total_num_edges"]}) for x in r_time_ls] + time_ls.extend(r_time_ls) + + print( + f"Benchmark completed for replication factor = {replication_factor}\n{'=' * 30}", + flush=True, + ) + + df = pd.DataFrame(time_ls) + df.to_csv("cugraph_dgl_e2e_benchmark.csv", index=False) + print(f"Benchmark completed for all replication factors\n{'=' * 30}", flush=True) + + +def e2e_benchmark( + sampled_dir: str, feat: torch.Tensor, y: torch.Tensor, sampled_meta_d: dict +): + """ + Run the e2e_benchmark + Args: + sampled_dir: directory containing the sampled graph + feat: node features + y: node labels + sampled_meta_d: dictionary containing the sampled graph metadata + """ + time_ls = [] + + # TODO: Make this a parameter in bulk sampling script + sampled_meta_d["sparse_format"] = "csc" + sampled_dir = os.path.join(sampled_dir, "samples") + dataloader = create_dataloader( + sampled_dir, + sampled_meta_d["total_num_nodes"], + sampled_meta_d["sparse_format"], + return_type="cugraph_dgl.nn.SparseGraph", + ) + time_d = run_1_epoch( + dataloader, + feat, + y, + fanout=sampled_meta_d["fanout"], + batch_size=sampled_meta_d["batch_size"], + model_backend="cugraph_dgl", + ) + time_ls.append(time_d) + print("=" * 30) + return time_ls + + +def parse_arguments(): + parser = ArgumentParser() + parser.add_argument( + "--dataset_path", type=str, default="/raid/vjawa/ogbn_papers100M/" + ) + parser.add_argument( + "--sampling_path", + type=str, + default="/raid/vjawa/nov_1_bulksampling_benchmarks/", + ) + return parser.parse_args() + + +if __name__ == "__main__": + setup_common_pool() + arguments = parse_arguments() + main(arguments) diff --git a/benchmarks/cugraph-dgl/scale-benchmarks/dgl_benchmark.py b/benchmarks/cugraph-dgl/scale-benchmarks/dgl_benchmark.py index 3762226d570..3e0b8e3e0cc 100644 --- a/benchmarks/cugraph-dgl/scale-benchmarks/dgl_benchmark.py +++ b/benchmarks/cugraph-dgl/scale-benchmarks/dgl_benchmark.py @@ -22,6 +22,7 @@ from argparse import ArgumentParser from load_graph_feats import load_edges_from_disk, load_node_labels, load_node_features + class DataLoaderArgs: def __init__(self, args): self.dataset_path = args.dataset_path @@ -31,7 +32,6 @@ def __init__(self, args): self.use_uva = not args.do_not_use_uva - def create_dataloader(g, train_idx, batch_size, fanouts, use_uva): print("Creating dataloader", flush=True) st = time.time() @@ -53,7 +53,6 @@ def create_dataloader(g, train_idx, batch_size, fanouts, use_uva): return dataloader - def create_dgl_graph_from_disk(dataset_path, replication_factor=1): """ Create a DGL graph from a dataset on disk. @@ -67,14 +66,14 @@ def create_dgl_graph_from_disk(dataset_path, replication_factor=1): input_meta = json.load(f) parquet_path = os.path.join(dataset_path, "parquet") - graph_data = load_edges_from_disk( - parquet_path, replication_factor, input_meta - ) + graph_data = load_edges_from_disk(parquet_path, replication_factor, input_meta) label_data = load_node_labels(dataset_path, replication_factor, input_meta) - if replication_factor <8 : - feat_data = load_node_features(dataset_path, replication_factor, node_type='paper') + if replication_factor < 8: + feat_data = load_node_features( + dataset_path, replication_factor, node_type="paper" + ) else: - feat_data = None + feat_data = None print("labels and features loaded ", flush=True) g = dgl.heterograph(graph_data) @@ -83,31 +82,49 @@ def create_dgl_graph_from_disk(dataset_path, replication_factor=1): def main(args): - print(f"Running dgl dataloading benchmark with the following parameters:\n" - f"Dataset path = {args.dataset_path}\n" - f"Replication factors = {args.replication_factors}\n" - f"Fanouts = {args.fanouts}\n" - f"Batch sizes = {args.batch_sizes}\n" - f"Use UVA = {args.use_uva}\n" - f"{'=' * 30}") + print( + f"Running dgl dataloading benchmark with the following parameters:\n" + f"Dataset path = {args.dataset_path}\n" + f"Replication factors = {args.replication_factors}\n" + f"Fanouts = {args.fanouts}\n" + f"Batch sizes = {args.batch_sizes}\n" + f"Use UVA = {args.use_uva}\n" + f"{'=' * 30}" + ) time_ls = [] for replication_factor in args.replication_factors: start_time = time.time() - g, label_data, feat_data = create_dgl_graph_from_disk(args.dataset_path, replication_factor) + g, label_data, feat_data = create_dgl_graph_from_disk( + args.dataset_path, replication_factor + ) elapsed_time = time.time() - start_time - print(f"Replication factor = {replication_factor}\n" - f"G has {g.num_edges():,} edges and took {elapsed_time:.2f} seconds to load", flush=True) + print( + f"Replication factor = {replication_factor}\n" + f"G has {g.num_edges():,} edges and took {elapsed_time:.2f} seconds to load", + flush=True, + ) train_idx = {"paper": label_data["paper"]["train_idx"]} y = label_data["paper"]["y"] - r_time_ls = e2e_benchmark(g, feat_data, y, train_idx, args.fanouts, args.batch_sizes, use_uva=args.use_uva) + r_time_ls = e2e_benchmark( + g, + feat_data, + y, + train_idx, + args.fanouts, + args.batch_sizes, + use_uva=args.use_uva, + ) [x.update({"replication_factor": replication_factor}) for x in r_time_ls] - [x.update({"num_edges": g.num_edges()}) for x in r_time_ls] + [x.update({"num_edges": g.num_edges()}) for x in r_time_ls] time_ls.extend(r_time_ls) - print(f"Benchmark completed for replication factor = {replication_factor}\n{'=' * 30}", flush=True) + print( + f"Benchmark completed for replication factor = {replication_factor}\n{'=' * 30}", + flush=True, + ) df = pd.DataFrame(time_ls) df.to_csv("dgl_e2e_benchmark.csv", index=False) @@ -131,22 +148,26 @@ def e2e_benchmark(g, feat, y, train_idx, fanouts, batch_sizes, use_uva): for fanout in fanouts: for batch_size in batch_sizes: dataloader = create_dataloader(g, train_idx, batch_size, fanout, use_uva) - time_d = run_1_epoch(dataloader, feat, y, fanout, batch_size, model_backend='dgl') + time_d = run_1_epoch( + dataloader, feat, y, fanout, batch_size, model_backend="dgl" + ) time_ls.append(time_d) - print("="*30) + print("=" * 30) return time_ls - def parse_arguments(): parser = ArgumentParser() - parser.add_argument("--dataset_path", type=str, default="/raid/vjawa/ogbn_papers100M") + parser.add_argument( + "--dataset_path", type=str, default="/raid/vjawa/ogbn_papers100M" + ) parser.add_argument("--replication_factors", type=str, default="2") parser.add_argument("--fanouts", type=str, default="10_10_10") parser.add_argument("--batch_sizes", type=str, default="512,1024,8192,16384") parser.add_argument("--do_not_use_uva", action="store_true") return parser.parse_args() + if __name__ == "__main__": arguments = parse_arguments() main(DataLoaderArgs(arguments)) diff --git a/benchmarks/cugraph-dgl/scale-benchmarks/load_graph_feats.py b/benchmarks/cugraph-dgl/scale-benchmarks/load_graph_feats.py index 4f0f81c70e1..f38d4432d46 100644 --- a/benchmarks/cugraph-dgl/scale-benchmarks/load_graph_feats.py +++ b/benchmarks/cugraph-dgl/scale-benchmarks/load_graph_feats.py @@ -30,17 +30,23 @@ def load_edges_from_disk(parquet_path, replication_factor, input_meta): graph_data = {} for edge_type in input_meta["num_edges"].keys(): - print(f"Loading edge index for edge type {edge_type} for replication factor = {replication_factor}") + print( + f"Loading edge index for edge type {edge_type} for replication factor = {replication_factor}" + ) canonical_edge_type = tuple(edge_type.split("__")) - edge_index = pd.read_parquet(os.path.join(parquet_path, edge_type, "edge_index.parquet")) + edge_index = pd.read_parquet( + os.path.join(parquet_path, edge_type, "edge_index.parquet") + ) edge_index = { "src": torch.from_numpy(edge_index.src.values), "dst": torch.from_numpy(edge_index.dst.values), } if replication_factor > 1: - src_list, dst_list = replicate_edges(edge_index, canonical_edge_type, replication_factor, input_meta) + src_list, dst_list = replicate_edges( + edge_index, canonical_edge_type, replication_factor, input_meta + ) edge_index["src"] = torch.cat(src_list).contiguous() edge_index["dst"] = torch.cat(dst_list).contiguous() @@ -55,26 +61,35 @@ def replicate_edges(edge_index, canonical_edge_type, replication_factor, input_m dst_list = [edge_index["dst"]] for r in range(1, replication_factor): - new_src = edge_index["src"] + (r * input_meta["num_nodes"][canonical_edge_type[0]]) - new_dst = edge_index["dst"] + (r * input_meta["num_nodes"][canonical_edge_type[2]]) + new_src = edge_index["src"] + ( + r * input_meta["num_nodes"][canonical_edge_type[0]] + ) + new_dst = edge_index["dst"] + ( + r * input_meta["num_nodes"][canonical_edge_type[2]] + ) src_list.append(new_src) dst_list.append(new_dst) return src_list, dst_list - - def load_node_labels(dataset_path, replication_factor, input_meta): - num_nodes_dict = {node_type: t * replication_factor for node_type, t in input_meta["num_nodes"].items()} + num_nodes_dict = { + node_type: t * replication_factor + for node_type, t in input_meta["num_nodes"].items() + } node_data = {} for node_type in input_meta["num_nodes"].keys(): node_data[node_type] = {} - label_path = os.path.join(dataset_path, "parquet", node_type, "node_label.parquet") + label_path = os.path.join( + dataset_path, "parquet", node_type, "node_label.parquet" + ) if os.path.exists(label_path): - node_data[node_type] = process_node_label(label_path, node_type, replication_factor, num_nodes_dict, input_meta) + node_data[node_type] = process_node_label( + label_path, node_type, replication_factor, num_nodes_dict, input_meta + ) else: node_data[node_type]["num_nodes"] = num_nodes_dict[node_type] @@ -82,30 +97,48 @@ def load_node_labels(dataset_path, replication_factor, input_meta): print("Loaded node labels", flush=True) return node_data -def process_node_label(label_path, node_type, replication_factor, num_nodes_dict, input_meta): + +def process_node_label( + label_path, node_type, replication_factor, num_nodes_dict, input_meta +): node_label = pd.read_parquet(label_path) if replication_factor > 1: - node_label = replicate_node_label(node_label, node_type, replication_factor, input_meta) + node_label = replicate_node_label( + node_label, node_type, replication_factor, input_meta + ) - node_label_tensor = torch.full((num_nodes_dict[node_type],), -1, dtype=torch.float32) - node_label_tensor[torch.as_tensor(node_label.node.values)] = torch.as_tensor(node_label.label.values) + node_label_tensor = torch.full( + (num_nodes_dict[node_type],), -1, dtype=torch.float32 + ) + node_label_tensor[torch.as_tensor(node_label.node.values)] = torch.as_tensor( + node_label.label.values + ) del node_label return { "train_idx": (node_label_tensor > -1).contiguous().nonzero().view(-1), - "y": node_label_tensor.contiguous().long() + "y": node_label_tensor.contiguous().long(), } def replicate_node_label(node_label, node_type, replication_factor, input_meta): base_num_nodes = input_meta["num_nodes"][node_type] - replicated_df = pd.DataFrame({ - "node": pd.concat([node_label.node + (r * base_num_nodes) for r in range(1, replication_factor)]), - "label": pd.concat([node_label.label for _ in range(1, replication_factor)]) - }) + replicated_df = pd.DataFrame( + { + "node": pd.concat( + [ + node_label.node + (r * base_num_nodes) + for r in range(1, replication_factor) + ] + ), + "label": pd.concat( + [node_label.label for _ in range(1, replication_factor)] + ), + } + ) return pd.concat([node_label, replicated_df]).reset_index(drop=True) @@ -114,10 +147,10 @@ def load_node_features(dataset_path, replication_factor, node_type): print("Loading node features", flush=True) node_type_path = os.path.join(dataset_path, "npy", node_type) if replication_factor == 1: - fname = os.path.join(node_type_path, "node_feat.npy") + fname = os.path.join(node_type_path, "node_feat.npy") else: fname = os.path.join(node_type_path, f"node_feat_{replication_factor}x.npy") - + feat = torch.from_numpy(np.load(fname)) print("Loaded node features", flush=True) return feat diff --git a/benchmarks/cugraph-dgl/scale-benchmarks/model.py b/benchmarks/cugraph-dgl/scale-benchmarks/model.py index 506e3bd5227..9a9dfe58f96 100644 --- a/benchmarks/cugraph-dgl/scale-benchmarks/model.py +++ b/benchmarks/cugraph-dgl/scale-benchmarks/model.py @@ -17,8 +17,15 @@ class GNN(torch.nn.Module): - def __init__(self, in_channels, hidden_channels, out_channels, num_layers, model_backend='dgl'): - if model_backend == 'dgl': + def __init__( + self, + in_channels, + hidden_channels, + out_channels, + num_layers, + model_backend="dgl", + ): + if model_backend == "dgl": from dgl.nn import SAGEConv else: from cugraph_dgl.nn import SAGEConv @@ -26,9 +33,13 @@ def __init__(self, in_channels, hidden_channels, out_channels, num_layers, model super(GNN, self).__init__() self.convs = torch.nn.ModuleList() for _ in range(num_layers - 1): - self.convs.append(SAGEConv(in_channels, hidden_channels, aggregator_type='mean')) + self.convs.append( + SAGEConv(in_channels, hidden_channels, aggregator_type="mean") + ) in_channels = hidden_channels - self.convs.append(SAGEConv(hidden_channels, out_channels, aggregator_type='mean')) + self.convs.append( + SAGEConv(hidden_channels, out_channels, aggregator_type="mean") + ) def forward(self, blocks, x): for i, conv in enumerate(self.convs): @@ -38,44 +49,47 @@ def forward(self, blocks, x): return x -def create_model(feat_size, num_classes, num_layers, model_backend='dgl'): +def create_model(feat_size, num_classes, num_layers, model_backend="dgl"): model = GNN(feat_size, 64, num_classes, num_layers, model_backend=model_backend) - model = model.to('cuda') + model = model.to("cuda") model.train() return model + def train_model(model, dataloader, opt, feat, y): - times = {key: 0 for key in ['mfg_creation', 'feature', 'm_fwd', 'm_bkwd']} + times_d = {key: 0 for key in ["mfg_creation", "feature", "m_fwd", "m_bkwd"]} epoch_st = time.time() mfg_st = time.time() for input_nodes, output_nodes, blocks in dataloader: - times['mfg_creation'] += time.time() - mfg_st + times_d["mfg_creation"] += time.time() - mfg_st if feat is not None: fst = time.time() - input_nodes = input_nodes.to('cpu') + input_nodes = input_nodes.to("cpu") input_feat = feat[input_nodes] - input_feat = input_feat.to('cuda') + input_feat = input_feat.to("cuda") if isinstance(output_nodes, dict): - output_nodes = output_nodes['paper'] + output_nodes = output_nodes["paper"] output_nodes = output_nodes.to(y.device) - y_batch = y[output_nodes].to('cuda') - times['feature'] += time.time() - fst + y_batch = y[output_nodes].to("cuda") + times_d["feature"] += time.time() - fst m_fwd_st = time.time() y_hat = model(blocks, input_feat) - times['m_fwd'] += time.time() - m_fwd_st - + times_d["m_fwd"] += time.time() - m_fwd_st + m_bkwd_st = time.time() loss = F.cross_entropy(y_hat, y_batch) opt.zero_grad() loss.backward() opt.step() - times['m_bkwd'] += time.time() - m_bkwd_st + times_d["m_bkwd"] += time.time() - m_bkwd_st mfg_st = time.time() print(f"Epoch time = {time.time() - epoch_st:.2f} seconds") - - return times + print(f"Time to create MFG = {times_d['mfg_creation']:.2f} seconds") + + return times_d + def analyze_time(dataloader, times, epoch_time, fanout, batch_size): num_batches = len(dataloader) @@ -92,17 +106,24 @@ def analyze_time(dataloader, times, epoch_time, fanout, batch_size): print(f"Time analysis for fanout = {fanout}, batch_size = {batch_size}") for k in time_d.keys(): - if 'time_per_epoch' in str(k): + if "time_per_epoch" in str(k): print(f"{k} = {time_d[k]:.2f} seconds") return time_d + def run_1_epoch(dataloader, feat, y, fanout, batch_size, model_backend): if feat is not None: - model = create_model(feat.shape[1], 172, len(fanout), model_backend=model_backend) + model = create_model( + feat.shape[1], 172, len(fanout), model_backend=model_backend + ) opt = torch.optim.Adam(model.parameters(), lr=0.01) else: model = None opt = None + + # Warmup RUN + times = train_model(model, dataloader, opt, feat, y) + epoch_st = time.time() times = train_model(model, dataloader, opt, feat, y) epoch_time = time.time() - epoch_st diff --git a/benchmarks/cugraph-service/pytest-based/bench_cgs_client_scaling.py b/benchmarks/cugraph-service/pytest-based/bench_cgs_client_scaling.py index 9871ce64a31..541b3e23490 100644 --- a/benchmarks/cugraph-service/pytest-based/bench_cgs_client_scaling.py +++ b/benchmarks/cugraph-service/pytest-based/bench_cgs_client_scaling.py @@ -46,11 +46,8 @@ def running_server_for_sampling_with_graph(request): control returns to this fixture, the graph is deleted. If the fixture started a server subprocess, then it is terminated as well. """ - (client, server_process) = ( - utils.ensure_running_server_for_sampling(host=_host, - port=_port, - dask_scheduler_file=None, - start_local_cuda_cluster=True) + (client, server_process) = utils.ensure_running_server_for_sampling( + host=_host, port=_port, dask_scheduler_file=None, start_local_cuda_cluster=True ) num_edges = (2**_graph_scale) * _edge_factor gid = client.call_graph_creation_extension( @@ -80,9 +77,7 @@ def create_sampling_client(host, port, graph_id): """ client = CugraphServiceClient(host, port) fanout_vals = [10, 25] - start_list = client.call_extension( - "gen_vertex_list", graph_id, _batch_size - ) + start_list = client.call_extension("gen_vertex_list", graph_id, _batch_size) def sampling_function(result_device): return client.uniform_neighbor_sample( @@ -108,9 +103,11 @@ def sampling_function(result_device): # Use the benchmark fixture once it can be run in a way where each run is timed # with other running clients. + @pytest.mark.parametrize("num_clients", params.num_clients.values()) -def bench_cgs_client_scaling_individual_time(running_server_for_sampling_with_graph, - num_clients): +def bench_cgs_client_scaling_individual_time( + running_server_for_sampling_with_graph, num_clients +): graph_id = running_server_for_sampling_with_graph @@ -133,8 +130,9 @@ def bench_cgs_client_scaling_individual_time(running_server_for_sampling_with_gr @pytest.mark.parametrize("num_clients", params.num_clients.values()) -def bench_cgs_client_scaling_total_time(running_server_for_sampling_with_graph, - num_clients): +def bench_cgs_client_scaling_total_time( + running_server_for_sampling_with_graph, num_clients +): graph_id = running_server_for_sampling_with_graph diff --git a/benchmarks/cugraph/pytest-based/bench_algos.py b/benchmarks/cugraph/pytest-based/bench_algos.py index d7fcb7812e4..69d6700ca3d 100644 --- a/benchmarks/cugraph/pytest-based/bench_algos.py +++ b/benchmarks/cugraph/pytest-based/bench_algos.py @@ -14,13 +14,16 @@ import pytest import numpy as np import pytest_benchmark + # FIXME: Remove this when rapids_pytest_benchmark.gpubenchmark is available # everywhere try: from rapids_pytest_benchmark import setFixtureParamNames except ImportError: - print("\n\nWARNING: rapids_pytest_benchmark is not installed, " - "falling back to pytest_benchmark fixtures.\n") + print( + "\n\nWARNING: rapids_pytest_benchmark is not installed, " + "falling back to pytest_benchmark fixtures.\n" + ) # if rapids_pytest_benchmark is not available, just perfrom time-only # benchmarking and replace the util functions with nops @@ -29,6 +32,7 @@ def setFixtureParamNames(*args, **kwargs): pass + import rmm import dask_cudf from pylibcugraph.testing import gen_fixture_params_product @@ -65,7 +69,7 @@ def get_edgelist(self, fetch=False): if self._edgelist is None: self._edgelist = rmat( self._scale, - (2**self._scale)*self._edgefactor, + (2**self._scale) * self._edgefactor, 0.57, # from Graph500 0.19, # from Graph500 0.19, # from Graph500 @@ -73,22 +77,25 @@ def get_edgelist(self, fetch=False): clip_and_flip=False, scramble_vertex_ids=True, create_using=None, # return edgelist instead of Graph instance - mg=self.mg + mg=self.mg, ) rng = np.random.default_rng(seed) if self.mg: self._edgelist["weight"] = self._edgelist.map_partitions( - lambda df: rng.random(size=len(df))) + lambda df: rng.random(size=len(df)) + ) else: self._edgelist["weight"] = rng.random(size=len(self._edgelist)) return self._edgelist - def get_graph(self, - fetch=False, - create_using=cugraph.Graph, - ignore_weights=False, - store_transposed=False): + def get_graph( + self, + fetch=False, + create_using=cugraph.Graph, + ignore_weights=False, + store_transposed=False, + ): if isinstance(create_using, cugraph.Graph): # what about BFS if trnaposed is True attrs = {"directed": create_using.is_directed()} @@ -99,17 +106,21 @@ def get_graph(self, edge_attr = None if ignore_weights else "weight" df = self.get_edgelist() if isinstance(df, dask_cudf.DataFrame): - G.from_dask_cudf_edgelist(df, - source="src", - destination="dst", - edge_attr=edge_attr, - store_transposed=store_transposed) + G.from_dask_cudf_edgelist( + df, + source="src", + destination="dst", + edge_attr=edge_attr, + store_transposed=store_transposed, + ) else: - G.from_cudf_edgelist(df, - source="src", - destination="dst", - edge_attr=edge_attr, - store_transposed=store_transposed) + G.from_cudf_edgelist( + df, + source="src", + destination="dst", + edge_attr=edge_attr, + store_transposed=store_transposed, + ) return G def get_path(self): @@ -125,26 +136,27 @@ def unload(self): _rmat_scale = getattr(pytest, "_rmat_scale", 20) # ~1M vertices _rmat_edgefactor = getattr(pytest, "_rmat_edgefactor", 16) # ~17M edges -rmat_sg_dataset = pytest.param(RmatDataset(scale=_rmat_scale, - edgefactor=_rmat_edgefactor, - mg=False), - marks=[pytest.mark.rmat_data, - pytest.mark.sg, - ]) -rmat_mg_dataset = pytest.param(RmatDataset(scale=_rmat_scale, - edgefactor=_rmat_edgefactor, - mg=True), - marks=[pytest.mark.rmat_data, - pytest.mark.mg, - ]) +rmat_sg_dataset = pytest.param( + RmatDataset(scale=_rmat_scale, edgefactor=_rmat_edgefactor, mg=False), + marks=[ + pytest.mark.rmat_data, + pytest.mark.sg, + ], +) +rmat_mg_dataset = pytest.param( + RmatDataset(scale=_rmat_scale, edgefactor=_rmat_edgefactor, mg=True), + marks=[ + pytest.mark.rmat_data, + pytest.mark.mg, + ], +) rmm_fixture_params = gen_fixture_params_product( - (managed_memory, "mm"), - (pool_allocator, "pa")) + (managed_memory, "mm"), (pool_allocator, "pa") +) dataset_fixture_params = gen_fixture_params_product( - (directed_datasets + - undirected_datasets + - [rmat_sg_dataset, rmat_mg_dataset], "ds")) + (directed_datasets + undirected_datasets + [rmat_sg_dataset, rmat_mg_dataset], "ds") +) # Record the current RMM settings so reinitialize() will be called only when a # change is needed (RMM defaults both values to False). The --allow-rmm-reinit @@ -153,8 +165,7 @@ def unload(self): # (see conftest.py for details). # The defaults for managed_mem (False) and pool_alloc (True) are set in # conftest.py -RMM_SETTINGS = {"managed_mem": False, - "pool_alloc": False} +RMM_SETTINGS = {"managed_mem": False, "pool_alloc": False} # FIXME: this only changes the RMM config in a SG environment. The dask config # that applies to RMM in an MG environment is not changed by this! @@ -163,16 +174,16 @@ def reinitRMM(managed_mem, pool_alloc): Reinitializes RMM to the value of managed_mem and pool_alloc, but only if those values are different that the current configuration. """ - if (managed_mem != RMM_SETTINGS["managed_mem"]) or \ - (pool_alloc != RMM_SETTINGS["pool_alloc"]): + if (managed_mem != RMM_SETTINGS["managed_mem"]) or ( + pool_alloc != RMM_SETTINGS["pool_alloc"] + ): rmm.reinitialize( managed_memory=managed_mem, pool_allocator=pool_alloc, - initial_pool_size=2 << 27 + initial_pool_size=2 << 27, ) - RMM_SETTINGS.update(managed_mem=managed_mem, - pool_alloc=pool_alloc) + RMM_SETTINGS.update(managed_mem=managed_mem, pool_alloc=pool_alloc) ############################################################################### @@ -185,8 +196,8 @@ def reinitRMM(managed_mem, pool_alloc): # For benchmarks, the operations performed in fixtures are not measured as part # of the benchmark. -@pytest.fixture(scope="module", - params=rmm_fixture_params) + +@pytest.fixture(scope="module", params=rmm_fixture_params) def rmm_config(request): # Since parameterized fixtures do not assign param names to param values, # manually call the helper to do so. Ensure the order of the name list @@ -196,8 +207,7 @@ def rmm_config(request): reinitRMM(request.param[0], request.param[1]) -@pytest.fixture(scope="module", - params=dataset_fixture_params) +@pytest.fixture(scope="module", params=dataset_fixture_params) def dataset(request, rmm_config): """ @@ -261,27 +271,29 @@ def is_graph_distributed(graph): ############################################################################### # Benchmarks def bench_create_graph(gpubenchmark, edgelist): - gpubenchmark(cugraph.from_cudf_edgelist, - edgelist, - source="src", destination="dst", - create_using=cugraph.structure.graph_classes.Graph, - renumber=False) + gpubenchmark( + cugraph.from_cudf_edgelist, + edgelist, + source="src", + destination="dst", + create_using=cugraph.structure.graph_classes.Graph, + renumber=False, + ) # Creating directed Graphs on small datasets runs in micro-seconds, which # results in thousands of rounds before the default threshold is met, so lower # the max_time for this benchmark. -@pytest.mark.benchmark( - warmup=True, - warmup_iterations=10, - max_time=0.005 -) +@pytest.mark.benchmark(warmup=True, warmup_iterations=10, max_time=0.005) def bench_create_digraph(gpubenchmark, edgelist): - gpubenchmark(cugraph.from_cudf_edgelist, - edgelist, - source="src", destination="dst", - create_using=cugraph.Graph(directed=True), - renumber=False) + gpubenchmark( + cugraph.from_cudf_edgelist, + edgelist, + source="src", + destination="dst", + create_using=cugraph.Graph(directed=True), + renumber=False, + ) def bench_renumber(gpubenchmark, edgelist): @@ -289,8 +301,11 @@ def bench_renumber(gpubenchmark, edgelist): def bench_pagerank(gpubenchmark, transposed_graph): - pagerank = dask_cugraph.pagerank if is_graph_distributed(transposed_graph) \ - else cugraph.pagerank + pagerank = ( + dask_cugraph.pagerank + if is_graph_distributed(transposed_graph) + else cugraph.pagerank + ) gpubenchmark(pagerank, transposed_graph) @@ -319,7 +334,8 @@ def bench_jaccard(gpubenchmark, unweighted_graph): @pytest.mark.skipif( - is_device_version_less_than((7, 0)), reason="Not supported on Pascal") + is_device_version_less_than((7, 0)), reason="Not supported on Pascal" +) def bench_louvain(gpubenchmark, graph): louvain = dask_cugraph.louvain if is_graph_distributed(graph) else cugraph.louvain gpubenchmark(louvain, graph) @@ -342,8 +358,11 @@ def bench_overlap(gpubenchmark, unweighted_graph): def bench_triangle_count(gpubenchmark, graph): - tc = dask_cugraph.triangle_count if is_graph_distributed(graph) \ - else cugraph.triangle_count + tc = ( + dask_cugraph.triangle_count + if is_graph_distributed(graph) + else cugraph.triangle_count + ) gpubenchmark(tc, graph) @@ -353,12 +372,13 @@ def bench_spectralBalancedCutClustering(gpubenchmark, graph): gpubenchmark(cugraph.spectralBalancedCutClustering, graph, 2) -@pytest.mark.skip(reason="Need to guarantee graph has weights, " - "not doing that yet") +@pytest.mark.skip(reason="Need to guarantee graph has weights, " "not doing that yet") def bench_spectralModularityMaximizationClustering(gpubenchmark, graph): - smmc = dask_cugraph.spectralModularityMaximizationClustering \ - if is_graph_distributed(graph) \ - else cugraph.spectralModularityMaximizationClustering + smmc = ( + dask_cugraph.spectralModularityMaximizationClustering + if is_graph_distributed(graph) + else cugraph.spectralModularityMaximizationClustering + ) gpubenchmark(smmc, graph, 2) @@ -373,8 +393,11 @@ def bench_graph_degrees(gpubenchmark, graph): def bench_betweenness_centrality(gpubenchmark, graph): - bc = dask_cugraph.betweenness_centrality if is_graph_distributed(graph) \ - else cugraph.betweenness_centrality + bc = ( + dask_cugraph.betweenness_centrality + if is_graph_distributed(graph) + else cugraph.betweenness_centrality + ) gpubenchmark(bc, graph, k=10, random_state=123) @@ -385,8 +408,11 @@ def bench_edge_betweenness_centrality(gpubenchmark, graph): def bench_uniform_neighbor_sample(gpubenchmark, graph): - uns = dask_cugraph.uniform_neighbor_sample if is_graph_distributed(graph) \ - else cugraph.uniform_neighbor_sample + uns = ( + dask_cugraph.uniform_neighbor_sample + if is_graph_distributed(graph) + else cugraph.uniform_neighbor_sample + ) seed = 42 # FIXME: may need to provide number_of_vertices separately @@ -405,8 +431,9 @@ def bench_uniform_neighbor_sample(gpubenchmark, graph): def bench_egonet(gpubenchmark, graph): - egonet = dask_cugraph.ego_graph if is_graph_distributed(graph) \ - else cugraph.ego_graph + egonet = ( + dask_cugraph.ego_graph if is_graph_distributed(graph) else cugraph.ego_graph + ) n = 1 radius = 2 gpubenchmark(egonet, graph, n, radius=radius) diff --git a/benchmarks/cugraph/pytest-based/conftest.py b/benchmarks/cugraph/pytest-based/conftest.py index fd029471869..57329366a13 100644 --- a/benchmarks/cugraph/pytest-based/conftest.py +++ b/benchmarks/cugraph/pytest-based/conftest.py @@ -15,31 +15,37 @@ def pytest_addoption(parser): - parser.addoption("--allow-rmm-reinit", - action="store_true", - default=False, - help="Allow RMM to be reinitialized, possibly multiple times within " - "the same process, in order to run benchmarks with different managed " - "memory and pool allocator options. This is not the default behavior " - "since it does not represent a typical use case, and support for " - "this may be limited. Instead, consider multiple pytest runs that " - "use a fixed set of RMM settings.") - parser.addoption("--rmat-scale", - action="store", - type=int, - default=20, - metavar="scale", - help="For use when using synthetic graph data generated using RMAT. " - "This results in a graph with 2^scale vertices. Default is " - "%(default)s.") - parser.addoption("--rmat-edgefactor", - action="store", - type=int, - default=16, - metavar="edgefactor", - help="For use when using synthetic graph data generated using RMAT. " - "This results in a graph with (2^scale)*edgefactor edges. Default " - "is %(default)s.") + parser.addoption( + "--allow-rmm-reinit", + action="store_true", + default=False, + help="Allow RMM to be reinitialized, possibly multiple times within " + "the same process, in order to run benchmarks with different managed " + "memory and pool allocator options. This is not the default behavior " + "since it does not represent a typical use case, and support for " + "this may be limited. Instead, consider multiple pytest runs that " + "use a fixed set of RMM settings.", + ) + parser.addoption( + "--rmat-scale", + action="store", + type=int, + default=20, + metavar="scale", + help="For use when using synthetic graph data generated using RMAT. " + "This results in a graph with 2^scale vertices. Default is " + "%(default)s.", + ) + parser.addoption( + "--rmat-edgefactor", + action="store", + type=int, + default=16, + metavar="edgefactor", + help="For use when using synthetic graph data generated using RMAT. " + "This results in a graph with (2^scale)*edgefactor edges. Default " + "is %(default)s.", + ) def pytest_sessionstart(session): @@ -57,10 +63,11 @@ def pytest_sessionstart(session): if session.config.getoption("allow_rmm_reinit") is False: currentMarkexpr = session.config.getoption("markexpr") - if ("managedmem" in currentMarkexpr) or \ - ("poolallocator" in currentMarkexpr): - raise RuntimeError("managedmem and poolallocator markers cannot " - "be used without --allow-rmm-reinit.") + if ("managedmem" in currentMarkexpr) or ("poolallocator" in currentMarkexpr): + raise RuntimeError( + "managedmem and poolallocator markers cannot " + "be used without --allow-rmm-reinit." + ) newMarkexpr = "managedmem_off and poolallocator_on" if currentMarkexpr: diff --git a/benchmarks/cugraph/standalone/benchmark.py b/benchmarks/cugraph/standalone/benchmark.py index e02194648a6..74b73d9d8ff 100644 --- a/benchmarks/cugraph/standalone/benchmark.py +++ b/benchmarks/cugraph/standalone/benchmark.py @@ -21,6 +21,7 @@ class BenchmarkedResult: Class to hold results (the return value of the callable being benchmarked and meta-data about the benchmarked function) of a benchmarked function run. """ + def __init__(self, name, retval, runtime, params=None): self.name = name self.retval = retval @@ -45,16 +46,18 @@ def benchmark(func): functions to benchmark. """ benchmark_name = getattr(func, "benchmark_name", func.__name__) + @wraps(func) def benchmark_wrapper(*func_args, **func_kwargs): t1 = time.perf_counter() retval = func(*func_args, **func_kwargs) t2 = time.perf_counter() - return BenchmarkedResult(name=benchmark_name, - retval=retval, - runtime=(t2-t1), - params=func_kwargs, - ) + return BenchmarkedResult( + name=benchmark_name, + retval=retval, + runtime=(t2 - t1), + params=func_kwargs, + ) # Assign the name to the returned callable as well for use in debug prints, # etc. @@ -68,17 +71,21 @@ class BenchmarkRun: method, and results are saved as BenchmarkedResult instances in the results list member. """ - def __init__(self, - input_dataframe, - construct_graph_func, - algo_func_param_list, - algo_validator_list=None - ): + + def __init__( + self, + input_dataframe, + construct_graph_func, + algo_func_param_list, + algo_validator_list=None, + ): self.input_dataframe = input_dataframe if type(construct_graph_func) is tuple: - (construct_graph_func, - self.construct_graph_func_args) = construct_graph_func + ( + construct_graph_func, + self.construct_graph_func_args, + ) = construct_graph_func else: self.construct_graph_func_args = None @@ -89,15 +96,15 @@ def __init__(self, # add starting node to algos: BFS and SSSP # FIXME: Refactor BenchmarkRun __init__ because all the work # done below should be done elsewhere - for i, algo in enumerate (algo_func_param_list): + for i, algo in enumerate(algo_func_param_list): if benchmark(algo).name in ["bfs", "sssp", "uniform_neighbor_sample"]: - param={} - param["start"]=self.input_dataframe['src'].head()[0] + param = {} + param["start"] = self.input_dataframe["src"].head()[0] if benchmark(algo).name in ["uniform_neighbor_sample"]: start = [param.pop("start")] param["start_list"] = start param["fanout_vals"] = [1] - algo_func_param_list[i]=(algo,)+(param,) + algo_func_param_list[i] = (algo,) + (param,) self.algos = [] for item in algo_func_param_list: @@ -110,13 +117,11 @@ def __init__(self, self.validators = algo_validator_list or [None] * len(self.algos) self.results = [] - @staticmethod def __log(s, end="\n"): print(s, end=end) sys.stdout.flush() - def run(self): """ Run and time the graph construction step, then run and time each algo. @@ -124,8 +129,9 @@ def run(self): self.results = [] self.__log(f"running {self.construct_graph.name}...", end="") - result = self.construct_graph(self.input_dataframe, - *self.construct_graph_func_args) + result = self.construct_graph( + self.input_dataframe, *self.construct_graph_func_args + ) self.__log("done.") G = result.retval self.results.append(result) @@ -141,22 +147,21 @@ def run(self): if self.construct_graph.name == "from_dask_cudf_edgelist": # compute out_degree before renumbering because out_degree # has transpose=False - degree_max = G.degree()['degree'].max().compute() + degree_max = G.degree()["degree"].max().compute() katz_alpha = 1 / (degree_max) self.algos[i][1]["alpha"] = katz_alpha elif self.construct_graph.name == "from_cudf_edgelist": - degree_max = G.degree()['degree'].max() + degree_max = G.degree()["degree"].max() katz_alpha = 1 / (degree_max) self.algos[i][1]["alpha"] = katz_alpha if hasattr(G, "compute_renumber_edge_list"): - G.compute_renumber_edge_list( - transposed=True) + G.compute_renumber_edge_list(transposed=True) else: # FIXME: Pagerank still follows the old path. Update this once it # follows the pylibcugraph/C path if hasattr(G, "compute_renumber_edge_list"): G.compute_renumber_edge_list(transposed=True) - else: #set transpose=False when renumbering + else: # set transpose=False when renumbering self.__log("running compute_renumber_edge_list...", end="") if hasattr(G, "compute_renumber_edge_list"): if self.algos[i][0].name in ["wcc", "louvain"]: @@ -164,8 +169,7 @@ def run(self): # Update this once it follows the pylibcugraph/C path G.compute_renumber_edge_list(transposed=False) else: - G.compute_renumber_edge_list( - transposed=False) + G.compute_renumber_edge_list(transposed=False) self.__log("done.") # FIXME: need to handle individual algo args for ((algo, params), validator) in zip(self.algos, self.validators): diff --git a/benchmarks/cugraph/standalone/bulk_sampling/cugraph_bulk_sampling.py b/benchmarks/cugraph/standalone/bulk_sampling/cugraph_bulk_sampling.py index d2a3716da8a..1ca5d6db637 100644 --- a/benchmarks/cugraph/standalone/bulk_sampling/cugraph_bulk_sampling.py +++ b/benchmarks/cugraph/standalone/bulk_sampling/cugraph_bulk_sampling.py @@ -22,7 +22,6 @@ get_allocation_counts_dask_lazy, sizeof_fmt, get_peak_output_ratio_across_workers, - restart_client, start_dask_client, stop_dask_client, enable_spilling, @@ -66,19 +65,19 @@ def construct_graph(dask_dataframe): Returns: G: cugraph.Graph """ - assert dask_dataframe['src'].dtype == 'int64' - assert dask_dataframe['dst'].dtype == 'int64' + assert dask_dataframe["src"].dtype == "int64" + assert dask_dataframe["dst"].dtype == "int64" - if 'etp' in dask_dataframe.columns: - assert dask_dataframe['etp'].dtype == 'int32' + if "etp" in dask_dataframe.columns: + assert dask_dataframe["etp"].dtype == "int32" G = cugraph.MultiGraph(directed=True) G.from_dask_cudf_edgelist( dask_dataframe, - source="src", + source="src", destination="dst", - edge_type='etp' if 'etp' in dask_dataframe.columns else None, - renumber=False + edge_type="etp" if "etp" in dask_dataframe.columns else None, + renumber=False, ) return G @@ -86,66 +85,87 @@ def construct_graph(dask_dataframe): def symmetrize_ddf(dask_dataframe): source_col, dest_col = symmetrize( dask_dataframe, - 'src', - 'dst', + "src", + "dst", multi=True, symmetrize=True, ) new_ddf = source_col.to_frame() - new_ddf['dst'] = dest_col + new_ddf["dst"] = dest_col return new_ddf + def renumber_ddf(dask_df, persist=False): - vertices = dask_cudf.concat([dask_df['src'], dask_df['dst']]).unique().reset_index(drop=True) + vertices = ( + dask_cudf.concat([dask_df["src"], dask_df["dst"]]) + .unique() + .reset_index(drop=True) + ) if persist: vertices = vertices.persist() - - vertices.name = 'v' - vertices = vertices.reset_index().set_index('v').rename(columns={'index': 'm'}) + + vertices.name = "v" + vertices = vertices.reset_index().set_index("v").rename(columns={"index": "m"}) if persist: vertices = vertices.persist() - src = dask_df.merge(vertices, left_on='src', right_on='v', how='left').m.rename('src') - dst = dask_df.merge(vertices, left_on='dst', right_on='v', how='left').m.rename('dst') + src = dask_df.merge(vertices, left_on="src", right_on="v", how="left").m.rename( + "src" + ) + dst = dask_df.merge(vertices, left_on="dst", right_on="v", how="left").m.rename( + "dst" + ) df = src.to_frame() - df['dst'] = dst + df["dst"] = dst return df.reset_index(drop=True) -def _make_batch_ids(bdf: cudf.DataFrame, batch_size: int, num_workers: int, partition_info: Optional[Union[dict, str]] = None): + +def _make_batch_ids( + bdf: cudf.DataFrame, + batch_size: int, + num_workers: int, + partition_info: Optional[Union[dict, str]] = None, +): # Required by dask; need to skip dummy partitions. if partition_info is None: - return cudf.DataFrame({ - 'batch': cudf.Series(dtype='int32'), - 'start': cudf.Series(dtype='int64') - }) - - partition = partition_info['number'] + return cudf.DataFrame( + {"batch": cudf.Series(dtype="int32"), "start": cudf.Series(dtype="int64")} + ) + + partition = partition_info["number"] if partition is None: - raise ValueError('division is absent') + raise ValueError("division is absent") num_batches = int(ceil(len(bdf) / batch_size)) - + batch_ids = cupy.repeat( - cupy.arange(num_batches * partition, num_batches * (partition + 1), dtype='int32'), - batch_size - )[:len(bdf)] + cupy.arange( + num_batches * partition, num_batches * (partition + 1), dtype="int32" + ), + batch_size, + )[: len(bdf)] bdf = bdf.reset_index(drop=True) - bdf['batch'] = cudf.Series(batch_ids) + bdf["batch"] = cudf.Series(batch_ids) return bdf -def _replicate_df(df: cudf.DataFrame, replication_factor: int, col_item_counts:Dict[str, int], partition_info: Optional[Union[dict, str]] = None): +def _replicate_df( + df: cudf.DataFrame, + replication_factor: int, + col_item_counts: Dict[str, int], + partition_info: Optional[Union[dict, str]] = None, +): # Required by dask; need to skip dummy partitions. if partition_info is None: - return cudf.DataFrame({ - col: cudf.Series(dtype=df[col].dtype) for col in col_item_counts.keys() - }) - + return cudf.DataFrame( + {col: cudf.Series(dtype=df[col].dtype) for col in col_item_counts.keys()} + ) + original_df = df.copy() if replication_factor > 1: @@ -153,14 +173,24 @@ def _replicate_df(df: cudf.DataFrame, replication_factor: int, col_item_counts:D df_replicated = original_df for col, offset in col_item_counts.items(): df_replicated[col] += offset * r - + df = cudf.concat([df, df_replicated], ignore_index=True) - + return df @get_allocation_counts_dask_lazy(return_allocations=True, logging=True) -def sample_graph(G, label_df, output_path,seed=42, batch_size=500, seeds_per_call=200000, batches_per_partition=100, fanout=[5, 5, 5], persist=False): +def sample_graph( + G, + label_df, + output_path, + seed=42, + batch_size=500, + seeds_per_call=400000, + batches_per_partition=100, + fanout=[5, 5, 5], + sampling_kwargs={}, +): cupy.random.seed(seed) sampler = BulkSampler( @@ -172,35 +202,36 @@ def sample_graph(G, label_df, output_path,seed=42, batch_size=500, seeds_per_cal random_state=seed, seeds_per_call=seeds_per_call, batches_per_partition=batches_per_partition, - log_level = logging.INFO + log_level=logging.INFO, + **sampling_kwargs, ) - n_workers = len(default_client().scheduler_info()['workers']) + n_workers = len(default_client().scheduler_info()["workers"]) + + meta = cudf.DataFrame( + {"node": cudf.Series(dtype="int64"), "batch": cudf.Series(dtype="int32")} + ) - meta = cudf.DataFrame({ - 'node': cudf.Series(dtype='int64'), - 'batch': cudf.Series(dtype='int32') - }) + batch_df = label_df.map_partitions( + _make_batch_ids, batch_size, n_workers, meta=meta + ) + # batch_df = batch_df.sort_values(by='node') - batch_df = label_df.map_partitions(_make_batch_ids, batch_size, n_workers, meta=meta) - #batch_df = batch_df.sort_values(by='node') - # should always persist the batch dataframe or performance may be suboptimal batch_df = batch_df.persist() del label_df - print('created batches') - + print("created batches") start_time = perf_counter() - sampler.add_batches(batch_df, start_col_name='node', batch_col_name='batch') + sampler.add_batches(batch_df, start_col_name="node", batch_col_name="batch") sampler.flush() end_time = perf_counter() - print('flushed all batches') - return (end_time - start_time) + print("flushed all batches") + return end_time - start_time -def assign_offsets_pyg(node_counts: Dict[str, int], replication_factor:int=1): +def assign_offsets_pyg(node_counts: Dict[str, int], replication_factor: int = 1): # cuGraph-PyG assigns offsets based on lexicographic order node_offsets = {} node_offsets_replicated = {} @@ -212,10 +243,19 @@ def assign_offsets_pyg(node_counts: Dict[str, int], replication_factor:int=1): count += node_counts[node_type] count_replicated += node_counts[node_type] * replication_factor - + return node_offsets, node_offsets_replicated, count_replicated -def generate_rmat_dataset(dataset, seed=62, labeled_percentage=0.01, num_labels=256, reverse_edges=False, persist=False, add_edge_types=False): + +def generate_rmat_dataset( + dataset, + seed=62, + labeled_percentage=0.01, + num_labels=256, + reverse_edges=False, + persist=False, + add_edge_types=False, +): """ Generates an rmat dataset. Currently does not support heterogeneous datasets. @@ -227,17 +267,20 @@ def generate_rmat_dataset(dataset, seed=62, labeled_percentage=0.01, num_labels= reverse_edges: Whether to reverse the edges in the edgelist (should be True for DGL, False, for PyG) """ - dataset = dataset.split('_') + dataset = dataset.split("_") scale = int(dataset[1]) edgefactor = int(dataset[2]) dask_edgelist_df = generate_edgelist_rmat( - scale=scale, edgefactor=edgefactor, seed=seed, unweighted=True, mg=True, + scale=scale, + edgefactor=edgefactor, + seed=seed, + unweighted=True, + mg=True, ) dask_edgelist_df = dask_edgelist_df.astype("int64") dask_edgelist_df = dask_edgelist_df.reset_index(drop=True) - dask_edgelist_df = renumber_ddf(dask_edgelist_df).persist() if persist: dask_edgelist_df = dask_edgelist_df.persist() @@ -247,96 +290,126 @@ def generate_rmat_dataset(dataset, seed=62, labeled_percentage=0.01, num_labels= dask_edgelist_df = dask_edgelist_df.persist() if add_edge_types: - dask_edgelist_df['etp'] = cupy.int32(0) # doesn't matter what the value is, really - + dask_edgelist_df["etp"] = cupy.int32( + 0 + ) # doesn't matter what the value is, really + # generator = np.random.default_rng(seed=seed) - num_labeled_nodes = int(2**(scale+1) * labeled_percentage) - label_df = pd.DataFrame({ - 'node': np.arange(num_labeled_nodes), - # 'label': generator.integers(0, num_labels - 1, num_labeled_nodes).astype('float32') - }) - - n_workers = len(default_client().scheduler_info()['workers']) - dask_label_df = ddf.from_pandas(label_df, npartitions=n_workers*2) + num_labeled_nodes = int(2 ** (scale + 1) * labeled_percentage) + label_df = pd.DataFrame( + { + "node": np.arange(num_labeled_nodes), + # 'label': generator.integers(0, num_labels - 1, num_labeled_nodes).astype('float32') + } + ) + + n_workers = len(default_client().scheduler_info()["workers"]) + dask_label_df = ddf.from_pandas(label_df, npartitions=n_workers * 2) del label_df gc.collect() dask_label_df = dask_cudf.from_dask_dataframe(dask_label_df) - node_offsets = {'paper': 0} - edge_offsets = {('paper','cites','paper'):0} - total_num_nodes = int(dask_cudf.concat([dask_edgelist_df.src, dask_edgelist_df.dst]).nunique().compute()) + node_offsets = {"paper": 0} + edge_offsets = {("paper", "cites", "paper"): 0} + total_num_nodes = int( + dask_cudf.concat([dask_edgelist_df.src, dask_edgelist_df.dst]) + .nunique() + .compute() + ) if reverse_edges: - dask_edgelist_df = dask_edgelist_df.rename(columns={'src':'dst', 'dst':'src'}) + dask_edgelist_df = dask_edgelist_df.rename(columns={"src": "dst", "dst": "src"}) return dask_edgelist_df, dask_label_df, node_offsets, edge_offsets, total_num_nodes -def load_disk_dataset(dataset, dataset_dir='.', reverse_edges=True, replication_factor=1, persist=False, add_edge_types=False): +def load_disk_dataset( + dataset, + dataset_dir=".", + reverse_edges=True, + replication_factor=1, + persist=False, + add_edge_types=False, +): from pathlib import Path + path = Path(dataset_dir) / dataset - parquet_path = path / 'parquet' + parquet_path = path / "parquet" n_workers = get_n_workers() - with open(os.path.join(path, 'meta.json')) as meta_file: + with open(os.path.join(path, "meta.json")) as meta_file: meta = json.load(meta_file) - - node_offsets, node_offsets_replicated, total_num_nodes = \ - assign_offsets_pyg(meta['num_nodes'], replication_factor=replication_factor) + + node_offsets, node_offsets_replicated, total_num_nodes = assign_offsets_pyg( + meta["num_nodes"], replication_factor=replication_factor + ) edge_index_dict = {} - for edge_type in meta['num_edges'].keys(): - print(f'Loading edge index for edge type {edge_type}') + for edge_type in meta["num_edges"].keys(): + print(f"Loading edge index for edge type {edge_type}") - can_edge_type = tuple(edge_type.split('__')) + can_edge_type = tuple(edge_type.split("__")) edge_index_dict[can_edge_type] = dask_cudf.read_parquet( - Path(parquet_path) / edge_type / 'edge_index.parquet' - ).repartition(n_workers*2) + Path(parquet_path) / edge_type / "edge_index.parquet" + ).repartition(n_workers * 2) - edge_index_dict[can_edge_type]['src'] += node_offsets_replicated[can_edge_type[0]] - edge_index_dict[can_edge_type]['dst'] += node_offsets_replicated[can_edge_type[-1]] + edge_index_dict[can_edge_type]["src"] += node_offsets_replicated[ + can_edge_type[0] + ] + edge_index_dict[can_edge_type]["dst"] += node_offsets_replicated[ + can_edge_type[-1] + ] edge_index_dict[can_edge_type] = edge_index_dict[can_edge_type] if persist: edge_index_dict = edge_index_dict.persist() if replication_factor > 1: - edge_index_dict[can_edge_type] = edge_index_dict[can_edge_type].map_partitions( + edge_index_dict[can_edge_type] = edge_index_dict[ + can_edge_type + ].map_partitions( _replicate_df, replication_factor, { - 'src': meta['num_nodes'][can_edge_type[0]], - 'dst': meta['num_nodes'][can_edge_type[2]], + "src": meta["num_nodes"][can_edge_type[0]], + "dst": meta["num_nodes"][can_edge_type[2]], }, - meta=cudf.DataFrame({'src':cudf.Series(dtype='int64'), 'dst':cudf.Series(dtype='int64')}) + meta=cudf.DataFrame( + { + "src": cudf.Series(dtype="int64"), + "dst": cudf.Series(dtype="int64"), + } + ), ) - + if persist: - edge_index_dict[can_edge_type] = edge_index_dict[can_edge_type].persist() - + edge_index_dict[can_edge_type] = edge_index_dict[ + can_edge_type + ].persist() + gc.collect() if reverse_edges: - edge_index_dict[can_edge_type] = edge_index_dict[can_edge_type].rename(columns={'src':'dst','dst':'src'}) - + edge_index_dict[can_edge_type] = edge_index_dict[can_edge_type].rename( + columns={"src": "dst", "dst": "src"} + ) + if persist: edge_index_dict[can_edge_type] = edge_index_dict[can_edge_type].persist() - + # Assign numeric edge type ids based on lexicographic order edge_offsets = {} edge_count = 0 for num_edge_type, can_edge_type in enumerate(sorted(edge_index_dict.keys())): if add_edge_types: - edge_index_dict[can_edge_type]['etp'] = cupy.int32(num_edge_type) + edge_index_dict[can_edge_type]["etp"] = cupy.int32(num_edge_type) edge_offsets[can_edge_type] = edge_count edge_count += len(edge_index_dict[can_edge_type]) - - all_edges_df = dask_cudf.concat( - list(edge_index_dict.values()) - ) - + + all_edges_df = dask_cudf.concat(list(edge_index_dict.values())) + if persist: all_edges_df = all_edges_df.persist() @@ -345,55 +418,66 @@ def load_disk_dataset(dataset, dataset_dir='.', reverse_edges=True, replication_ node_labels = {} for node_type, offset in node_offsets_replicated.items(): - print(f'Loading node labels for node type {node_type} (offset={offset})') - node_label_path = os.path.join(os.path.join(parquet_path, node_type), 'node_label.parquet') + print(f"Loading node labels for node type {node_type} (offset={offset})") + node_label_path = os.path.join( + os.path.join(parquet_path, node_type), "node_label.parquet" + ) if os.path.exists(node_label_path): - node_labels[node_type] = dask_cudf.read_parquet(node_label_path).repartition(n_workers).drop('label',axis=1).persist() - node_labels[node_type]['node'] += offset + node_labels[node_type] = ( + dask_cudf.read_parquet(node_label_path) + .repartition(n_workers) + .drop("label", axis=1) + .persist() + ) + node_labels[node_type]["node"] += offset node_labels[node_type] = node_labels[node_type].persist() if replication_factor > 1: node_labels[node_type] = node_labels[node_type].map_partitions( _replicate_df, replication_factor, - { - 'node': meta['num_nodes'][node_type] - }, - meta=cudf.DataFrame({'node':cudf.Series(dtype='int64')}) + {"node": meta["num_nodes"][node_type]}, + meta=cudf.DataFrame({"node": cudf.Series(dtype="int64")}), ) - + if persist: node_labels[node_type] = node_labels[node_type].persist() gc.collect() - - node_labels_df = dask_cudf.concat( - list(node_labels.values()) - ) - + + node_labels_df = dask_cudf.concat(list(node_labels.values())) + if persist: node_labels_df = node_labels_df.persist() del node_labels gc.collect() - return all_edges_df, node_labels_df, node_offsets_replicated, edge_offsets, total_num_nodes - + return ( + all_edges_df, + node_labels_df, + node_offsets_replicated, + edge_offsets, + total_num_nodes, + ) + def benchmark_cugraph_bulk_sampling( - dataset, - output_path, - seed, - batch_size, - seeds_per_call, - fanout, - reverse_edges=True, - dataset_dir='.', - replication_factor=1, - num_labels=256, - labeled_percentage=0.001, - persist=False, - add_edge_types=False): + dataset, + output_path, + seed, + batch_size, + seeds_per_call, + fanout, + sampling_target_framework, + reverse_edges=True, + dataset_dir=".", + replication_factor=1, + num_labels=256, + labeled_percentage=0.001, + persist=False, + add_edge_types=False, +): """ Entry point for the benchmark. @@ -430,87 +514,113 @@ def benchmark_cugraph_bulk_sampling( Defaults to False. """ print(dataset) - if dataset[0:4] == 'rmat': - dask_edgelist_df, dask_label_df, node_offsets, edge_offsets, total_num_nodes = \ - generate_rmat_dataset( - dataset, - reverse_edges=reverse_edges, - seed=seed, - labeled_percentage=labeled_percentage, - num_labels=num_labels, - persist=persist, - add_edge_types=add_edge_types - ) + if dataset[0:4] == "rmat": + ( + dask_edgelist_df, + dask_label_df, + node_offsets, + edge_offsets, + total_num_nodes, + ) = generate_rmat_dataset( + dataset, + reverse_edges=reverse_edges, + seed=seed, + labeled_percentage=labeled_percentage, + num_labels=num_labels, + persist=persist, + add_edge_types=add_edge_types, + ) else: - dask_edgelist_df, dask_label_df, node_offsets, edge_offsets, total_num_nodes = \ - load_disk_dataset( - dataset, - dataset_dir=dataset_dir, - reverse_edges=reverse_edges, - replication_factor=replication_factor, - persist=persist, - add_edge_types=add_edge_types - ) + ( + dask_edgelist_df, + dask_label_df, + node_offsets, + edge_offsets, + total_num_nodes, + ) = load_disk_dataset( + dataset, + dataset_dir=dataset_dir, + reverse_edges=reverse_edges, + replication_factor=replication_factor, + persist=persist, + add_edge_types=add_edge_types, + ) num_input_edges = len(dask_edgelist_df) - print( - f"Number of input edges = {num_input_edges:,}" - ) + print(f"Number of input edges = {num_input_edges:,}") - G = construct_graph( - dask_edgelist_df - ) + G = construct_graph(dask_edgelist_df) del dask_edgelist_df - print('constructed graph') + print("constructed graph") input_memory = G.edgelist.edgelist_df.memory_usage().sum().compute() - print(f'input memory: {input_memory}') + print(f"input memory: {input_memory}") - output_subdir = os.path.join(output_path, f'{dataset}[{replication_factor}]_b{batch_size}_f{fanout}') + output_subdir = os.path.join( + output_path, f"{dataset}[{replication_factor}]_b{batch_size}_f{fanout}" + ) os.makedirs(output_subdir) - output_sample_path = os.path.join(output_subdir, 'samples') + output_sample_path = os.path.join(output_subdir, "samples") os.makedirs(output_sample_path) - batches_per_partition = 200_000 // batch_size + if sampling_target_framework == "cugraph_dgl_csr": + sampling_kwargs = { + "deduplicate_sources": True, + "prior_sources_behavior": "carryover", + "renumber": True, + "compression": "CSR", + "compress_per_hop": True, + "use_legacy_names": False, + "include_hop_column": False, + } + else: + # FIXME: Update these arguments when CSC mode is fixed in cuGraph-PyG (release 24.02) + sampling_kwargs = { + "deduplicate_sources": True, + "prior_sources_behavior": "exclude", + "renumber": True, + "compression": "COO", + "compress_per_hop": False, + "use_legacy_names": False, + "include_hop_column": True, + } + + batches_per_partition = 400_000 // batch_size execution_time, allocation_counts = sample_graph( - G, - dask_label_df, - output_sample_path, + G=G, + label_df=dask_label_df, + output_path=output_sample_path, seed=seed, batch_size=batch_size, seeds_per_call=seeds_per_call, batches_per_partition=batches_per_partition, fanout=fanout, - persist=persist, + sampling_kwargs=sampling_kwargs, ) output_meta = { - 'dataset': dataset, - 'dataset_dir': dataset_dir, - 'seed': seed, - 'node_offsets': node_offsets, - 'edge_offsets': {'__'.join(k): v for k, v in edge_offsets.items()}, - 'total_num_nodes': total_num_nodes, - 'total_num_edges': num_input_edges, - 'batch_size': batch_size, - 'seeds_per_call': seeds_per_call, - 'batches_per_partition': batches_per_partition, - 'fanout': fanout, - 'replication_factor': replication_factor, - 'num_sampling_gpus': len(G._plc_graph), - 'execution_time': execution_time, + "dataset": dataset, + "dataset_dir": dataset_dir, + "seed": seed, + "node_offsets": node_offsets, + "edge_offsets": {"__".join(k): v for k, v in edge_offsets.items()}, + "total_num_nodes": total_num_nodes, + "total_num_edges": num_input_edges, + "batch_size": batch_size, + "seeds_per_call": seeds_per_call, + "batches_per_partition": batches_per_partition, + "fanout": fanout, + "replication_factor": replication_factor, + "num_sampling_gpus": len(G._plc_graph), + "execution_time": execution_time, } - with open(os.path.join(output_subdir, 'output_meta.json'), 'w') as f: - json.dump( - output_meta, - f, - indent='\t' - ) + with open(os.path.join(output_subdir, "output_meta.json"), "w") as f: + json.dump(output_meta, f, indent="\t") - print('allocation counts b:') + print("allocation counts b:") print(allocation_counts.values()) ( @@ -558,91 +668,93 @@ def get_memory_statistics(allocation_counts, input_memory): def get_args(): parser = argparse.ArgumentParser() - + parser.add_argument( - '--output_root', + "--output_root", type=str, - help='The output root directory. File/folder names are auto-generated.', + help="The output root directory. File/folder names are auto-generated.", required=True, ) parser.add_argument( - '--dataset_root', + "--dataset_root", type=str, - help='The dataset root directory containing ogb datasets.', + help="The dataset root directory containing ogb datasets.", required=True, ) parser.add_argument( - '--datasets', + "--datasets", type=str, help=( - 'Comma separated list of datasets; can specify ogb or rmat (i.e. ogb_papers100M[2],rmat_22_16).' - ' For ogb datasets, can provide replication factor using brackets.' + "Comma separated list of datasets; can specify ogb or rmat (i.e. ogb_papers100M[2],rmat_22_16)." + " For ogb datasets, can provide replication factor using brackets." ), required=True, ) parser.add_argument( - '--fanouts', + "--fanouts", type=str, - help='Comma separated list of fanouts (i.e. 10_25,5_5_5)', + help="Comma separated list of fanouts (i.e. 10_25,5_5_5)", required=False, - default='10_25', + default="10_25", ) parser.add_argument( - '--batch_sizes', + "--batch_sizes", type=str, - help='Comma separated list of batch sizes (i.e. 500,1000)', + help="Comma separated list of batch sizes (i.e. 500,1000)", required=False, - default='512,1024' + default="512,1024", ) parser.add_argument( - '--seeds_per_call_opts', + "--seeds_per_call_opts", type=str, - help='Comma separated list of seeds per call (i.e. 1000000,2000000)', + help="Comma separated list of seeds per call (i.e. 1000000,2000000)", required=False, - default='524288', + default="524288", ) - + parser.add_argument( - '--reverse_edges', - action='store_true', - help='Whether to reverse the edges for DGL (defaults to False). Should be True for DGL, False for PyG.', + "--reverse_edges", + action="store_true", + help="Whether to reverse the edges for DGL (defaults to False). Should be True for DGL, False for PyG.", required=False, default=False, ) - parser.add_argument( - '--dask_worker_devices', + "--sampling_target_framework", type=str, - help='Comma separated list of dask worker devices', + help="The target framework for sampling (i.e. cugraph_dgl_csr, cugraph_pyg_csc, ...)", required=False, - default="0" + default=None, ) - parser.add_argument( - '--random_seed', - type=int, - help='Random seed', + "--dask_worker_devices", + type=str, + help="Comma separated list of dask worker devices", required=False, - default=62 + default="0", + ) + + parser.add_argument( + "--random_seed", type=int, help="Random seed", required=False, default=62 ) parser.add_argument( - '--persist', - action='store_true', - help='Will add additional persist() calls to speed up ETL. Does not affect sampling runtime.', + "--persist", + action="store_true", + help="Will add additional persist() calls to speed up ETL. Does not affect sampling runtime.", required=False, default=False, ) parser.add_argument( - '--add_edge_types', - action='store_true', - help='Adds edge types to the edgelist. Required for PyG if not providing edge ids.', + "--add_edge_types", + action="store_true", + help="Adds edge types to the edgelist. Required for PyG if not providing edge ids.", required=False, default=False, ) @@ -655,18 +767,31 @@ def get_args(): logging.basicConfig() args = get_args() - fanouts = [[int(f) for f in fanout.split('_')] for fanout in args.fanouts.split(',')] - datasets = args.datasets.split(',') - batch_sizes = [int(b) for b in args.batch_sizes.split(',')] - seeds_per_call_opts = [int(s) for s in args.seeds_per_call_opts.split(',')] - dask_worker_devices = [int(d) for d in args.dask_worker_devices.split(',')] + if args.sampling_target_framework not in ["cugraph_dgl_csr", None]: + raise ValueError( + "sampling_target_framework must be one of cugraph_dgl_csr or None", + "Other frameworks are not supported at this time.", + ) - client, cluster = start_dask_client(dask_worker_devices=dask_worker_devices, jit_unspill=False, rmm_pool_size=28e9, rmm_async=True) + fanouts = [ + [int(f) for f in fanout.split("_")] for fanout in args.fanouts.split(",") + ] + datasets = args.datasets.split(",") + batch_sizes = [int(b) for b in args.batch_sizes.split(",")] + seeds_per_call_opts = [int(s) for s in args.seeds_per_call_opts.split(",")] + dask_worker_devices = [int(d) for d in args.dask_worker_devices.split(",")] + + client, cluster = start_dask_client( + dask_worker_devices=dask_worker_devices, + jit_unspill=False, + rmm_pool_size=28e9, + rmm_async=True, + ) enable_spilling() stats_ls = [] client.run(enable_spilling) for dataset in datasets: - if re.match(r'([A-z]|[0-9])+\[[0-9]+\]', dataset): + if re.match(r"([A-z]|[0-9])+\[[0-9]+\]", dataset): replication_factor = int(dataset[-2]) dataset = dataset[:-3] else: @@ -675,10 +800,10 @@ def get_args(): for fanout in fanouts: for batch_size in batch_sizes: for seeds_per_call in seeds_per_call_opts: - print(f'dataset: {dataset}') - print(f'batch size: {batch_size}') - print(f'fanout: {fanout}') - print(f'seeds_per_call: {seeds_per_call}') + print(f"dataset: {dataset}") + print(f"batch size: {batch_size}") + print(f"fanout: {fanout}") + print(f"seeds_per_call: {seeds_per_call}") try: stats_d = {} @@ -695,6 +820,7 @@ def get_args(): batch_size=batch_size, seeds_per_call=seeds_per_call, fanout=fanout, + sampling_target_framework=args.sampling_target_framework, dataset_dir=args.dataset_root, reverse_edges=args.reverse_edges, replication_factor=replication_factor, @@ -706,7 +832,9 @@ def get_args(): stats_d["batch_size"] = batch_size stats_d["fanout"] = fanout stats_d["seeds_per_call"] = seeds_per_call - stats_d["input_memory_per_worker"] = sizeof_fmt(input_memory_per_worker) + stats_d["input_memory_per_worker"] = sizeof_fmt( + input_memory_per_worker + ) stats_d["peak_allocation_across_workers"] = sizeof_fmt( peak_allocation_across_workers ) @@ -714,10 +842,9 @@ def get_args(): stats_d["output_to_peak_ratio"] = output_to_peak_ratio stats_ls.append(stats_d) except Exception as e: - warnings.warn('An Exception Occurred!') + warnings.warn("An Exception Occurred!") print(e) traceback.print_exc() - restart_client(client) sleep(10) stats_df = pd.DataFrame( diff --git a/benchmarks/cugraph/standalone/cugraph_dask_funcs.py b/benchmarks/cugraph/standalone/cugraph_dask_funcs.py index c6aa4a06100..e8f2c3a62bb 100644 --- a/benchmarks/cugraph/standalone/cugraph_dask_funcs.py +++ b/benchmarks/cugraph/standalone/cugraph_dask_funcs.py @@ -26,8 +26,6 @@ from cugraph.testing.mg_utils import generate_edgelist - - def read_csv(input_csv_file, scale): """ Returns a dask_cudf DataFrame from reading input_csv_file. @@ -41,15 +39,16 @@ def read_csv(input_csv_file, scale): """ vertex_t = "int32" if scale <= 32 else "int64" dtypes = [vertex_t, vertex_t, "float32"] - names=["src", "dst", "weight"], + names = (["src", "dst", "weight"],) chunksize = cugraph.dask.get_chunksize(input_csv_file) - return dask_cudf.read_csv(input_csv_file, - chunksize=chunksize, - delimiter=" ", - #names=names, - dtype=dtypes, - header=None, + return dask_cudf.read_csv( + input_csv_file, + chunksize=chunksize, + delimiter=" ", + # names=names, + dtype=dtypes, + header=None, ) @@ -59,6 +58,7 @@ def read_csv(input_csv_file, scale): # The "benchmark_name" attr is used by the benchmark infra for reporting and is # set to assign more meaningful names to be displayed in reports. + def construct_graph(dask_dataframe, symmetric=False): """ dask_dataframe contains weighted and undirected edges with self @@ -72,28 +72,27 @@ def construct_graph(dask_dataframe, symmetric=False): G = cugraph.Graph(directed=True) if len(dask_dataframe.columns) > 2: - if symmetric: #symmetrize dask dataframe - dask_dataframe = symmetrize_ddf( - dask_dataframe, 'src', 'dst', 'weight') + if symmetric: # symmetrize dask dataframe + dask_dataframe = symmetrize_ddf(dask_dataframe, "src", "dst", "weight") G.from_dask_cudf_edgelist( - dask_dataframe, source="src", destination="dst", edge_attr="weight") - #G.from_dask_cudf_edgelist( + dask_dataframe, source="src", destination="dst", edge_attr="weight" + ) + # G.from_dask_cudf_edgelist( # dask_dataframe, source="0", destination="1", edge_attr="2") else: - if symmetric: #symmetrize dask dataframe - dask_dataframe = symmetrize_ddf(dask_dataframe, 'src', 'dst') - G.from_dask_cudf_edgelist( - dask_dataframe, source="src", destination="dst") + if symmetric: # symmetrize dask dataframe + dask_dataframe = symmetrize_ddf(dask_dataframe, "src", "dst") + G.from_dask_cudf_edgelist(dask_dataframe, source="src", destination="dst") return G + construct_graph.benchmark_name = "from_dask_cudf_edgelist" def bfs(G, start): - return cugraph.dask.bfs( - G, start=start, return_distances=True, check_start=False) + return cugraph.dask.bfs(G, start=start, return_distances=True, check_start=False) def sssp(G, start): @@ -116,39 +115,49 @@ def katz(G, alpha=None): print(alpha) return cugraph.dask.katz_centrality(G, alpha) + def hits(G): return cugraph.dask.hits(G) + def uniform_neighbor_sample(G, start_list=None, fanout_vals=None): # convert list to cudf.Series - start_list = cudf.Series(start_list, dtype="int32") + start_list = cudf.Series(start_list, dtype="int32") return cugraph.dask.uniform_neighbor_sample( - G, start_list=start_list, fanout_vals=fanout_vals) + G, start_list=start_list, fanout_vals=fanout_vals + ) + def triangle_count(G): return cugraph.dask.triangle_count(G) + def eigenvector_centrality(G): return cugraph.dask.eigenvector_centrality(G) + ################################################################################ # Session-wide setup and teardown + def setup(dask_scheduler_file=None, rmm_pool_size=None): if dask_scheduler_file: cluster = None # Env var UCX_MAX_RNDV_RAILS=1 must be set too. - initialize(enable_tcp_over_ucx=True, - enable_nvlink=True, - enable_infiniband=False, - enable_rdmacm=False, - #net_devices="mlx5_0:1", - ) + initialize( + enable_tcp_over_ucx=True, + enable_nvlink=True, + enable_infiniband=False, + enable_rdmacm=False, + # net_devices="mlx5_0:1", + ) client = Client(scheduler_file=dask_scheduler_file) else: tempdir_object = tempfile.TemporaryDirectory() - cluster = LocalCUDACluster(local_directory=tempdir_object.name, rmm_pool_size=rmm_pool_size) + cluster = LocalCUDACluster( + local_directory=tempdir_object.name, rmm_pool_size=rmm_pool_size + ) client = Client(cluster) # add the obj to the client so it doesn't get deleted until # the 'client' obj gets cleaned up diff --git a/benchmarks/cugraph/standalone/cugraph_funcs.py b/benchmarks/cugraph/standalone/cugraph_funcs.py index ba33471a54f..d53471fa828 100644 --- a/benchmarks/cugraph/standalone/cugraph_funcs.py +++ b/benchmarks/cugraph/standalone/cugraph_funcs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -18,11 +18,12 @@ import cudf -def generate_edgelist(scale, - edgefactor, - seed=None, - unweighted=False, - ): +def generate_edgelist( + scale, + edgefactor, + seed=None, + unweighted=False, +): """ Returns a cudf DataFrame created using the R-MAT graph generator. @@ -43,7 +44,7 @@ def generate_edgelist(scale, """ df = rmat( scale, - (2**scale)*edgefactor, + (2**scale) * edgefactor, 0.57, # from Graph500 0.19, # from Graph500 0.19, # from Graph500 @@ -51,7 +52,7 @@ def generate_edgelist(scale, clip_and_flip=False, scramble_vertex_ids=True, create_using=None, # return edgelist instead of Graph instance - mg=False + mg=False, ) if not unweighted: rng = np.random.default_rng(seed) @@ -72,16 +73,17 @@ def read_csv(input_csv_file, scale): """ vertex_t = "int32" if scale <= 32 else "int64" dtypes = [vertex_t, vertex_t, "float32"] - names=["src", "dst", "weight"], + names = (["src", "dst", "weight"],) chunksize = cugraph.dask.get_chunksize(input_csv_file) - return cudf.read_csv(input_csv_file, - chunksize=chunksize, - delimiter=" ", - #names=names, - dtype=dtypes, - header=None, - ) + return cudf.read_csv( + input_csv_file, + chunksize=chunksize, + delimiter=" ", + # names=names, + dtype=dtypes, + header=None, + ) ################################################################################ @@ -90,6 +92,7 @@ def read_csv(input_csv_file, scale): # The "benchmark_name" attr is used by the benchmark infra for reporting and is # set to assign more meaningful names to be displayed in reports. + def construct_graph(dataframe, symmetric=False): """ dataframe contains weighted and undirected edges with self loops. Multiple @@ -103,15 +106,17 @@ def construct_graph(dataframe, symmetric=False): if len(dataframe.columns) > 2: G.from_cudf_edgelist( - dataframe, source="src", destination="dst", edge_attr="weight") - #G.from_cudf_edgelist( + dataframe, source="src", destination="dst", edge_attr="weight" + ) + # G.from_cudf_edgelist( # dataframe, source="0", destination="1", edge_attr="2") else: - G.from_cudf_edgelist( - dataframe, source="src", destination="dst") - #G.from_cudf_edgelist( + G.from_cudf_edgelist(dataframe, source="src", destination="dst") + # G.from_cudf_edgelist( # dataframe, source="0", destination="1") return G + + construct_graph.benchmark_name = "from_cudf_edgelist" @@ -138,24 +143,31 @@ def pagerank(G): def katz(G, alpha=None): return cugraph.katz_centrality(G, alpha) + def hits(G): return cugraph.hits(G) + def uniform_neighbor_sample(G, start_list=None, fanout_vals=None): # convert list to cudf.Series - start_list = cudf.Series(start_list, dtype="int32") + start_list = cudf.Series(start_list, dtype="int32") return cugraph.uniform_neighbor_sample( - G, start_list=start_list, fanout_vals=fanout_vals) + G, start_list=start_list, fanout_vals=fanout_vals + ) + def triangle_count(G): return cugraph.triangle_count(G) + def eigenvector_centrality(G): return cugraph.eigenvector_centrality(G) + ################################################################################ # Session-wide setup and teardown + def setup(*args, **kwargs): return tuple() diff --git a/benchmarks/cugraph/standalone/cugraph_graph_creation.py b/benchmarks/cugraph/standalone/cugraph_graph_creation.py index 1edf67bba44..70be0f8d470 100644 --- a/benchmarks/cugraph/standalone/cugraph_graph_creation.py +++ b/benchmarks/cugraph/standalone/cugraph_graph_creation.py @@ -210,7 +210,7 @@ def get_memory_statistics(allocation_counts, input_memory): print("-" * 40 + f"renumber completed" + "-" * 40) stats_df = pd.DataFrame( - stats_ls, + stats_ls, columns=[ "scale", "num_input_edges", diff --git a/benchmarks/cugraph/standalone/main.py b/benchmarks/cugraph/standalone/main.py index 206be6e9362..07e8eefde1d 100644 --- a/benchmarks/cugraph/standalone/main.py +++ b/benchmarks/cugraph/standalone/main.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -15,9 +15,10 @@ from cugraph.dask.common.mg_utils import get_visible_devices -from reporting import (generate_console_report, - update_csv_report, - ) +from reporting import ( + generate_console_report, + update_csv_report, +) import cugraph_funcs import cugraph_dask_funcs @@ -27,43 +28,45 @@ from pathlib import Path -def store_results_json(benchmark_dir=None, - algo_name=None, - algo_time=None, - n_gpus=None, - scale=None): +def store_results_json( + benchmark_dir=None, algo_name=None, algo_time=None, n_gpus=None, scale=None +): """ Store all benchmark results in json files """ benchmark_result = {} - benchmark_result['funcName'] = algo_name - benchmark_result['result'] = algo_time - benchmark_result['argNameValuePairs'] = [('scale', scale), ('ngpus', n_gpus)] + benchmark_result["funcName"] = algo_name + benchmark_result["result"] = algo_time + benchmark_result["argNameValuePairs"] = [("scale", scale), ("ngpus", n_gpus)] json_object = json.dumps(benchmark_result, indent=4) benchmark_dir_path = Path(benchmark_dir) - with open(f"{benchmark_dir_path}/benchmark_result_scale_{scale}_ngpus_{n_gpus}_{algo_name}.json", "w") as outfile: + with open( + f"{benchmark_dir_path}/benchmark_result_scale_{scale}_ngpus_{n_gpus}_{algo_name}.json", + "w", + ) as outfile: outfile.write(json_object) - def log(s, end="\n"): print(s, end=end) sys.stdout.flush() -def run(algos, - scale=None, - csv_graph_file=None, - csv_results_file=None, - unweighted=False, - symmetric=False, - edgefactor=None, - benchmark_dir=None, - dask_scheduler_file=None, - rmm_pool_size=None): +def run( + algos, + scale=None, + csv_graph_file=None, + csv_results_file=None, + unweighted=False, + symmetric=False, + edgefactor=None, + benchmark_dir=None, + dask_scheduler_file=None, + rmm_pool_size=None, +): """ Run the nightly benchmark on cugraph. Return True on success, False on failure. @@ -80,24 +83,24 @@ def run(algos, # Setup the benchmarks to run based on algos specified, or all. # Values are either callables, or tuples of (callable, args) pairs. - benchmarks = {"bfs": funcs.bfs, - "sssp": funcs.sssp, - "louvain": funcs.louvain, - "pagerank": funcs.pagerank, - "wcc": funcs.wcc, - "katz": funcs.katz, - "wcc": funcs.wcc, - "hits": funcs.hits, - "uniform_neighbor_sample": funcs.uniform_neighbor_sample, - "triangle_count": funcs.triangle_count, - "eigenvector_centrality": funcs.eigenvector_centrality, - } + benchmarks = { + "bfs": funcs.bfs, + "sssp": funcs.sssp, + "louvain": funcs.louvain, + "pagerank": funcs.pagerank, + "wcc": funcs.wcc, + "katz": funcs.katz, + "wcc": funcs.wcc, + "hits": funcs.hits, + "uniform_neighbor_sample": funcs.uniform_neighbor_sample, + "triangle_count": funcs.triangle_count, + "eigenvector_centrality": funcs.eigenvector_centrality, + } if algos: invalid_benchmarks = set(algos) - set(benchmarks.keys()) if invalid_benchmarks: - raise ValueError("Invalid benchmark(s) specified " - f"{invalid_benchmarks}") + raise ValueError("Invalid benchmark(s) specified " f"{invalid_benchmarks}") benchmarks_to_run = [benchmarks[b] for b in algos] else: benchmarks_to_run = list(benchmarks.values()) @@ -110,7 +113,7 @@ def run(algos, # If the number of GPUs is None, This is a MNMG run # Extract the number of gpus from the client if n_gpus is None: - n_gpus = len(setup_objs[0].scheduler_info()['workers']) + n_gpus = len(setup_objs[0].scheduler_info()["workers"]) log("done.") try: @@ -120,18 +123,18 @@ def run(algos, log("done.") elif scale: log("running generate_edgelist (RMAT)...", end="") - df = funcs.generate_edgelist(scale, - edgefactor=edgefactor, - seed=seed, - unweighted=unweighted) + df = funcs.generate_edgelist( + scale, edgefactor=edgefactor, seed=seed, unweighted=unweighted + ) log("done.") else: raise ValueError("Must specify either scale or csv_graph_file") - benchmark = BenchmarkRun(df, - (funcs.construct_graph, (symmetric,)), - benchmarks_to_run, - ) + benchmark = BenchmarkRun( + df, + (funcs.construct_graph, (symmetric,)), + benchmarks_to_run, + ) success = benchmark.run() algo_name = benchmark.results[1].name @@ -140,7 +143,7 @@ def run(algos, # Generate json files containing the benchmark results if benchmark_dir is not None: store_results_json(benchmark_dir, algo_name, algo_time, n_gpus, scale) - + # Report results print(generate_console_report(benchmark.results)) if csv_results_file: @@ -163,42 +166,75 @@ def run(algos, import argparse ap = argparse.ArgumentParser() - ap.add_argument("--scale", type=int, default=None, - help="scale factor for the graph edgelist generator " - "(num_verts=2**SCALE).") - ap.add_argument("--csv", type=str, default=None, - help="path to CSV file to read instead of generating a " - "graph edgelist.") - ap.add_argument("--unweighted", default=False, action="store_true", - help="Generate a graph without weights.") - ap.add_argument("--algo", action="append", - help="Algo to benchmark. May be specified multiple times. " - "Default is all algos.") - ap.add_argument("--dask-scheduler-file", type=str, default=None, - help="Dask scheduler file for multi-node configuration.") - ap.add_argument("--symmetric-graph", default=False, action="store_true", - help="Generate a symmetric (undirected) Graph instead of " - "a DiGraph.") - ap.add_argument("--edgefactor", type=int, default=16, - help="edge factor for the graph edgelist generator " - "(num_edges=num_verts*EDGEFACTOR).") - ap.add_argument("--benchmark-dir", type=str, default=None, - help="directory to store the results in json files") - ap.add_argument("--rmm-pool-size", type=str, default=None, - help="RMM pool size to initialize each worker with") - + ap.add_argument( + "--scale", + type=int, + default=None, + help="scale factor for the graph edgelist generator " "(num_verts=2**SCALE).", + ) + ap.add_argument( + "--csv", + type=str, + default=None, + help="path to CSV file to read instead of generating a " "graph edgelist.", + ) + ap.add_argument( + "--unweighted", + default=False, + action="store_true", + help="Generate a graph without weights.", + ) + ap.add_argument( + "--algo", + action="append", + help="Algo to benchmark. May be specified multiple times. " + "Default is all algos.", + ) + ap.add_argument( + "--dask-scheduler-file", + type=str, + default=None, + help="Dask scheduler file for multi-node configuration.", + ) + ap.add_argument( + "--symmetric-graph", + default=False, + action="store_true", + help="Generate a symmetric (undirected) Graph instead of " "a DiGraph.", + ) + ap.add_argument( + "--edgefactor", + type=int, + default=16, + help="edge factor for the graph edgelist generator " + "(num_edges=num_verts*EDGEFACTOR).", + ) + ap.add_argument( + "--benchmark-dir", + type=str, + default=None, + help="directory to store the results in json files", + ) + ap.add_argument( + "--rmm-pool-size", + type=str, + default=None, + help="RMM pool size to initialize each worker with", + ) args = ap.parse_args() - exitcode = run(algos=args.algo, - scale=args.scale, - csv_graph_file=args.csv, - csv_results_file="out.csv", - unweighted=args.unweighted, - symmetric=args.symmetric_graph, - edgefactor=args.edgefactor, - benchmark_dir=args.benchmark_dir, - dask_scheduler_file=args.dask_scheduler_file, - rmm_pool_size=args.rmm_pool_size) + exitcode = run( + algos=args.algo, + scale=args.scale, + csv_graph_file=args.csv, + csv_results_file="out.csv", + unweighted=args.unweighted, + symmetric=args.symmetric_graph, + edgefactor=args.edgefactor, + benchmark_dir=args.benchmark_dir, + dask_scheduler_file=args.dask_scheduler_file, + rmm_pool_size=args.rmm_pool_size, + ) sys.exit(exitcode) diff --git a/benchmarks/cugraph/standalone/pylibcugraph_bench.py b/benchmarks/cugraph/standalone/pylibcugraph_bench.py index eff3e2e8b7d..b958be8a4b8 100644 --- a/benchmarks/cugraph/standalone/pylibcugraph_bench.py +++ b/benchmarks/cugraph/standalone/pylibcugraph_bench.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -31,8 +31,9 @@ st = time.time() G2 = cugraph.Graph(directed=True) print(f"cugraph Graph create time: {time.time()-st}") -G2.from_cudf_edgelist(edgelist_df, source="src", destination="dst", - edge_attr="weight", renumber=True) +G2.from_cudf_edgelist( + edgelist_df, source="src", destination="dst", edge_attr="weight", renumber=True +) st = time.time() result = cugraph.pagerank(G2, alpha=0.85, tol=1.0e-6, max_iter=500) print(f"cugraph time: {time.time()-st}") @@ -42,27 +43,49 @@ resource_handle = pylibcugraph.experimental.ResourceHandle() graph_props = pylibcugraph.experimental.GraphProperties( - is_symmetric=False, is_multigraph=False) + is_symmetric=False, is_multigraph=False +) st = time.time() G = pylibcugraph.experimental.SGGraph( - resource_handle, graph_props, srcs, dsts, weights, - store_transposed=True, renumber=True, do_expensive_check=False) + resource_handle, + graph_props, + srcs, + dsts, + weights, + store_transposed=True, + renumber=True, + do_expensive_check=False, +) print(f"pylibcugraph Graph create time: {time.time()-st}") st = time.time() (vertices, pageranks) = pylibcugraph.experimental.pagerank( - resource_handle, G, None, alpha=0.85, epsilon=1.0e-6, max_iterations=500, - has_initial_guess=False, do_expensive_check=True) + resource_handle, + G, + None, + alpha=0.85, + epsilon=1.0e-6, + max_iterations=500, + has_initial_guess=False, + do_expensive_check=True, +) print(f"pylibcugraph time: {time.time()-st} (expensive check)") st = time.time() (vertices, pageranks) = pylibcugraph.experimental.pagerank( - resource_handle, G, None, alpha=0.85, epsilon=1.0e-6, max_iterations=500, - has_initial_guess=False, do_expensive_check=False) + resource_handle, + G, + None, + alpha=0.85, + epsilon=1.0e-6, + max_iterations=500, + has_initial_guess=False, + do_expensive_check=False, +) print(f"pylibcugraph time: {time.time()-st}") ######## print() vert_to_check = 4800348 -p = result['pagerank'][result['vertex'] == vert_to_check] +p = result["pagerank"][result["vertex"] == vert_to_check] print(f"cugraph pagerank for vert: {vert_to_check}: {p.iloc[0]}") host_verts = vertices.tolist() @@ -70,7 +93,7 @@ print(f"pylibcugraph pagerank for vert: {vert_to_check}: {pageranks[index]}") vert_to_check = 268434647 -p = result['pagerank'][result['vertex'] == vert_to_check] +p = result["pagerank"][result["vertex"] == vert_to_check] print(f"cugraph pagerank for vert: {vert_to_check}: {p.iloc[0]}") index = host_verts.index(vert_to_check) diff --git a/benchmarks/cugraph/standalone/reporting.py b/benchmarks/cugraph/standalone/reporting.py index afc04241384..e486ccf2c44 100644 --- a/benchmarks/cugraph/standalone/reporting.py +++ b/benchmarks/cugraph/standalone/reporting.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -56,6 +56,7 @@ def generate_console_report(benchmark_result_list): return retstring + def update_csv_report(csv_results_file, benchmark_result_list, ngpus): """ Update (or create if DNE) csv_results_file as a CSV file containing the @@ -93,9 +94,12 @@ def update_csv_report(csv_results_file, benchmark_result_list, ngpus): # Add any new algos that were not present in the existing CSV. If there was # no existing CSV, then this is all the algos ran in this run. for name in new_names - names_from_csv: - rows.append({"name": name, - ngpus_key: new_times[name], - }) + rows.append( + { + "name": name, + ngpus_key: new_times[name], + } + ) with open(csv_results_file, "w") as csv_file: field_names = sorted(all_fields) diff --git a/benchmarks/dgl/create_dataset.py b/benchmarks/dgl/create_dataset.py index c47d9068844..f002f79831c 100644 --- a/benchmarks/dgl/create_dataset.py +++ b/benchmarks/dgl/create_dataset.py @@ -15,6 +15,7 @@ import rmm from cugraph.generators import rmat + def create_dataset(scale, edgefactor, folder_path): _seed = 42 num_edges = (2**scale) * edgefactor @@ -31,14 +32,15 @@ def create_dataset(scale, edgefactor, folder_path): create_using=None, # None == return edgelist mg=False, ) - filepath = os.path.join(folder_path, f'rmat_scale_{scale}_edgefactor_{edgefactor}.parquet') + filepath = os.path.join( + folder_path, f"rmat_scale_{scale}_edgefactor_{edgefactor}.parquet" + ) edgelist_df.to_parquet(filepath) - -folder_path = os.path.join(os.getcwd(), 'datasets') +folder_path = os.path.join(os.getcwd(), "datasets") os.makedirs(folder_path, exist_ok=True) rmm.reinitialize(managed_memory=True) -for scale in [24,25,26]: +for scale in [24, 25, 26]: edgefactor = 16 create_dataset(scale=scale, edgefactor=edgefactor, folder_path=folder_path) diff --git a/benchmarks/dgl/pytest-based/dgl_benchmark.py b/benchmarks/dgl/pytest-based/dgl_benchmark.py index 93e50736359..456fa8fedc6 100644 --- a/benchmarks/dgl/pytest-based/dgl_benchmark.py +++ b/benchmarks/dgl/pytest-based/dgl_benchmark.py @@ -18,39 +18,59 @@ import torch import dgl -def get_edgelist(scale, edgefactor, dataset_dir = '../datasets'): - fp = os.path.join(dataset_dir, f'rmat_scale_{scale}_edgefactor_{edgefactor}.parquet') + +def get_edgelist(scale, edgefactor, dataset_dir="../datasets"): + fp = os.path.join( + dataset_dir, f"rmat_scale_{scale}_edgefactor_{edgefactor}.parquet" + ) return pd.read_parquet(fp) - + + def create_dgl_graph_from_df(df): - src_tensor = torch.as_tensor(df['src'].values) - dst_tensor = torch.as_tensor(df['dst'].values) + src_tensor = torch.as_tensor(df["src"].values) + dst_tensor = torch.as_tensor(df["dst"].values) # Reverse edges to match cuGraph behavior - g = dgl.graph(data = (dst_tensor, src_tensor)) + g = dgl.graph(data=(dst_tensor, src_tensor)) return g - -@pytest.mark.parametrize("scale_edge_factor", [[24,16],[25,16]]) -@pytest.mark.parametrize("batch_size", [100, 500, 1_000, 2_500, 5_000, 10_000, 20_000, 30_000, 40_000, 50_000, 60_000, 70_000, 80_000, 90_000, 100_000]) +@pytest.mark.parametrize("scale_edge_factor", [[24, 16], [25, 16]]) +@pytest.mark.parametrize( + "batch_size", + [ + 100, + 500, + 1_000, + 2_500, + 5_000, + 10_000, + 20_000, + 30_000, + 40_000, + 50_000, + 60_000, + 70_000, + 80_000, + 90_000, + 100_000, + ], +) @pytest.mark.parametrize("fanout", [[10, 25]]) def bench_dgl_pure_gpu(benchmark, scale_edge_factor, batch_size, fanout): df = get_edgelist(scale_edge_factor[0], scale_edge_factor[1]) - g = create_dgl_graph_from_df(df).to('cuda') - assert g.device.type =='cuda' - seed_nodes = torch.as_tensor(df['dst'][:batch_size]) - seed_nodes = seed_nodes.to('cuda') - assert len(seed_nodes)==batch_size + g = create_dgl_graph_from_df(df).to("cuda") + assert g.device.type == "cuda" + seed_nodes = torch.as_tensor(df["dst"][:batch_size]) + seed_nodes = seed_nodes.to("cuda") + assert len(seed_nodes) == batch_size ### Reverse because dgl sampler samples from destination to source fanout.reverse() sampler = dgl.dataloading.MultiLayerNeighborSampler(fanout) - + input_nodes, output_nodes, blocks = benchmark( - sampler.sample, - g, - seed_nodes=seed_nodes + sampler.sample, g, seed_nodes=seed_nodes ) - assert len(output_nodes)==batch_size + assert len(output_nodes) == batch_size return @@ -61,33 +81,55 @@ def dgl_graph_26_16(): return g, df -@pytest.mark.parametrize("batch_size", [100, 500, 1_000, 2_500, 5_000, 10_000, 20_000, 30_000, 40_000, 50_000, 60_000, 70_000, 80_000, 90_000, 100_000]) +@pytest.mark.parametrize( + "batch_size", + [ + 100, + 500, + 1_000, + 2_500, + 5_000, + 10_000, + 20_000, + 30_000, + 40_000, + 50_000, + 60_000, + 70_000, + 80_000, + 90_000, + 100_000, + ], +) @pytest.mark.parametrize("fanout", [[10, 25]]) def bench_dgl_uva(benchmark, dgl_graph_26_16, batch_size, fanout): - g,df = dgl_graph_26_16 - assert g.device.type =='cpu' - seed_nodes = torch.as_tensor(df['dst'][:30_000_000]) - seed_nodes = seed_nodes.to('cuda') + g, df = dgl_graph_26_16 + assert g.device.type == "cpu" + seed_nodes = torch.as_tensor(df["dst"][:30_000_000]) + seed_nodes = seed_nodes.to("cuda") ### Reverse because dgl sampler samples from destination to source fanout.reverse() sampler = dgl.dataloading.MultiLayerNeighborSampler(fanout) dataloader = dgl.dataloading.DataLoader( - g, - seed_nodes, # train_nid must be on GPU. - sampler, - device=torch.device('cuda:0'), # The device argument must be GPU. - num_workers=0, # Number of workers must be 0. - use_uva=True, - batch_size=batch_size, - drop_last=False, - shuffle=False) + g, + seed_nodes, # train_nid must be on GPU. + sampler, + device=torch.device("cuda:0"), # The device argument must be GPU. + num_workers=0, # Number of workers must be 0. + use_uva=True, + batch_size=batch_size, + drop_last=False, + shuffle=False, + ) def uva_benchmark(dataloader_it): input_nodes, output_nodes, blocks = next(dataloader_it) - return input_nodes, output_nodes, blocks + return input_nodes, output_nodes, blocks # added iterations and rounds to prevent dataloader going over num batches dataloader_it = iter(dataloader) - input_nodes, output_nodes, blocks = benchmark.pedantic(uva_benchmark,kwargs = {'dataloader_it': dataloader_it}, iterations=10, rounds=10) - assert len(output_nodes)==batch_size + input_nodes, output_nodes, blocks = benchmark.pedantic( + uva_benchmark, kwargs={"dataloader_it": dataloader_it}, iterations=10, rounds=10 + ) + assert len(output_nodes) == batch_size return diff --git a/benchmarks/nx-cugraph/pytest-based/bench_algos.py b/benchmarks/nx-cugraph/pytest-based/bench_algos.py new file mode 100644 index 00000000000..971c3ff1032 --- /dev/null +++ b/benchmarks/nx-cugraph/pytest-based/bench_algos.py @@ -0,0 +1,212 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import networkx as nx +import pandas as pd +import pytest +from cugraph import datasets + +# FIXME: promote these to cugraph.datasets so the following steps aren't +# necessary +# +# These datasets can be downloaded using the script in the 'datasets' dir: +# +# cd /datasets +# ./get_test_data.sh --benchmark +# +# Then set the following env var so the dataset utils can find their location: +# +# export RAPIDS_DATASET_ROOT_DIR=/datasets +# +from cugraph_benchmarking.params import ( + hollywood, + europe_osm, + cit_patents, + soc_livejournal, +) + +# Attempt to import the NetworkX dispatching module, which is only needed when +# testing with NX <3.2 in order to dynamically switch backends. NX >=3.2 allows +# the backend to be specified directly in the API call. +try: + from networkx.classes import backends # NX <3.2 +except ImportError: + backends = None + + +################################################################################ +# Fixtures and helpers +backend_params = ["cugraph", None] + +dataset_params = [ + pytest.param(datasets.karate, marks=[pytest.mark.small, pytest.mark.undirected]), + pytest.param(datasets.netscience, marks=[pytest.mark.small, pytest.mark.directed]), + pytest.param( + datasets.email_Eu_core, marks=[pytest.mark.small, pytest.mark.directed] + ), + pytest.param(cit_patents, marks=[pytest.mark.medium, pytest.mark.directed]), + pytest.param(hollywood, marks=[pytest.mark.medium, pytest.mark.undirected]), + pytest.param(europe_osm, marks=[pytest.mark.medium, pytest.mark.undirected]), + pytest.param(soc_livejournal, marks=[pytest.mark.large, pytest.mark.directed]), +] + + +def nx_graph_from_dataset(dataset_obj): + """ + Read the dataset specified by the dataset_obj and create and return a + nx.Graph or nx.DiGraph instance based on the dataset is_directed metadata. + """ + create_using = nx.DiGraph if dataset_obj.metadata["is_directed"] else nx.Graph + names = dataset_obj.metadata["col_names"] + dtypes = dataset_obj.metadata["col_types"] + if isinstance(dataset_obj.metadata["header"], int): + header = dataset_obj.metadata["header"] + else: + header = None + + pandas_edgelist = pd.read_csv( + dataset_obj.get_path(), + delimiter=dataset_obj.metadata["delim"], + names=names, + dtype=dict(zip(names, dtypes)), + header=header, + ) + G = nx.from_pandas_edgelist( + pandas_edgelist, source=names[0], target=names[1], create_using=create_using + ) + return G + + +# Test IDs are generated using the lambda assigned to the ids arg to provide an +# easier-to-read name from the Dataset obj string repr. +# See: https://docs.pytest.org/en/stable/reference/reference.html#pytest-fixture +@pytest.fixture(scope="module", params=dataset_params, ids=lambda ds: f"ds={str(ds)}") +def graph_obj(request): + """ + Returns a NX Graph or DiGraph obj from the dataset instance parameter. + """ + dataset = request.param + return nx_graph_from_dataset(dataset) + + +def get_legacy_backend_selector(backend_name): + """ + Returns a callable that wraps an algo function with either the default + dispatch decorator, or the "testing" decorator which unconditionally + dispatches. + This is only supported for NetworkX <3.2 + """ + backends.plugin_name = "cugraph" + orig_dispatch = backends._dispatch + testing_dispatch = backends.test_override_dispatch + + # Testing with the networkx <3.2 dispatch mechanism is based on decorating + # networkx APIs. The decorator is either one that only uses a backend if + # the input graph type is for that backend (the default decorator), or the + # "testing" decorator, which unconditionally converts a graph type to the + # type needed by the backend then calls the backend. If the cugraph backend + # is specified, create a callable that decorates the benchmarked function + # with the testing decorator. + # + # Because both the default and testing decorators assume they are only + # applied once and do bookkeeping to ensure algos are not registered + # multiple times, the callable also clears bookkeeping so the decorators + # can be reapplied multiple times. This is obviously a hack and networkx + # >=3.2 makes this use case properly supported. + if backend_name == "cugraph": + + def wrapper(*args, **kwargs): + backends._registered_algorithms = {} + return testing_dispatch(*args, **kwargs) + + else: + + def wrapper(*args, **kwargs): + backends._registered_algorithms = {} + return orig_dispatch(*args, **kwargs) + + return wrapper + + +def get_backend_selector(backend_name): + """ + Returns a callable that wraps an algo function in order to set the + "backend" kwarg on it. + This is only supported for NetworkX >= 3.2 + """ + + def get_callable_for_func(func): + def wrapper(*args, **kwargs): + kwargs["backend"] = backend_name + return func(*args, **kwargs) + + return wrapper + + return get_callable_for_func + + +@pytest.fixture( + scope="module", params=backend_params, ids=lambda backend: f"backend={backend}" +) +def backend_selector(request): + """ + Returns a callable that takes a function algo and wraps it in another + function that calls the algo using the appropriate backend. + """ + backend_name = request.param + if backends is not None: + return get_legacy_backend_selector(backend_name) + else: + return get_backend_selector(backend_name) + + +################################################################################ +# Benchmarks +normalized_params = [True, False] +k_params = [10, 100] + + +@pytest.mark.parametrize("normalized", normalized_params, ids=lambda norm: f"{norm=}") +@pytest.mark.parametrize("k", k_params, ids=lambda k: f"{k=}") +def bench_betweenness_centrality(benchmark, graph_obj, backend_selector, normalized, k): + result = benchmark( + backend_selector(nx.betweenness_centrality), + graph_obj, + weight=None, + normalized=normalized, + k=k, + ) + assert type(result) is dict + + +@pytest.mark.parametrize("normalized", normalized_params, ids=lambda norm: f"{norm=}") +def bench_edge_betweenness_centrality( + benchmark, graph_obj, backend_selector, normalized +): + result = benchmark( + backend_selector(nx.edge_betweenness_centrality), + graph_obj, + weight=None, + normalized=normalized, + ) + assert type(result) is dict + + +def bench_louvain_communities(benchmark, graph_obj, backend_selector): + # The cugraph backend for louvain_communities only supports undirected graphs + if isinstance(graph_obj, nx.DiGraph): + G = graph_obj.to_undirected() + else: + G = graph_obj + result = benchmark(backend_selector(nx.community.louvain_communities), G) + assert type(result) is list diff --git a/benchmarks/pytest.ini b/benchmarks/pytest.ini index 6af3aab27fe..b3d8a8bb36c 100644 --- a/benchmarks/pytest.ini +++ b/benchmarks/pytest.ini @@ -14,8 +14,10 @@ markers = managedmem_off: RMM managed memory disabled poolallocator_on: RMM pool allocator enabled poolallocator_off: RMM pool allocator disabled - small: small datasets tiny: tiny datasets + small: small datasets + medium: medium datasets + large: large datasets directed: directed datasets undirected: undirected datasets matrix_types: inputs are matrices diff --git a/benchmarks/shared/build_cugraph_ucx/test_client_bandwidth.py b/benchmarks/shared/build_cugraph_ucx/test_client_bandwidth.py index 681e304546a..3d52fb46421 100644 --- a/benchmarks/shared/build_cugraph_ucx/test_client_bandwidth.py +++ b/benchmarks/shared/build_cugraph_ucx/test_client_bandwidth.py @@ -20,51 +20,65 @@ import rmm from time import perf_counter_ns + def benchmark_func(func, n_times=10): def wrap_func(*args, **kwargs): time_ls = [] # ignore 1st run # and return other runs - for _ in range(0,n_times+1): + for _ in range(0, n_times + 1): t1 = perf_counter_ns() result = func(*args, **kwargs) t2 = perf_counter_ns() - time_ls.append(t2-t1) + time_ls.append(t2 - t1) return result, time_ls[1:] + return wrap_func + def create_dataframe(client): n_rows = 25_000_000 - df = cudf.DataFrame({'src':cp.arange(0,n_rows,dtype=cp.int32), 'dst':cp.arange(0,n_rows, dtype=cp.int32), 'eids':cp.ones(n_rows, cp.int32)}) - ddf = dask_cudf.from_cudf(df,npartitions= len(client.scheduler_info()['workers'])).persist() + df = cudf.DataFrame( + { + "src": cp.arange(0, n_rows, dtype=cp.int32), + "dst": cp.arange(0, n_rows, dtype=cp.int32), + "eids": cp.ones(n_rows, cp.int32), + } + ) + ddf = dask_cudf.from_cudf( + df, npartitions=len(client.scheduler_info()["workers"]) + ).persist() client.rebalance(ddf) del df _ = wait(ddf) return ddf + @benchmark_func def get_n_rows(ddf, n): - if n==-1: - df = ddf.compute() + if n == -1: + df = ddf.compute() else: df = ddf.head(n) return df + def run_bandwidth_test(ddf, n): df, time_ls = get_n_rows(ddf, n) time_ar = np.asarray(time_ls) time_mean = time_ar.mean() size_bytes = df.memory_usage().sum() - size_gb = round(size_bytes/(pow(1024,3)), 2) + size_gb = round(size_bytes / (pow(1024, 3)), 2) print(f"Getting {len(df):,} rows of size {size_gb} took = {time_mean*1e-6} ms") - time_mean_s = time_mean*1e-9 + time_mean_s = time_mean * 1e-9 print(f"Bandwidth = {round(size_gb/time_mean_s, 4)} gb/s") return - if __name__ == "__main__": - cluster = LocalCUDACluster(protocol='ucx',rmm_pool_size='15GB', CUDA_VISIBLE_DEVICES='1,2,3') + cluster = LocalCUDACluster( + protocol="ucx", rmm_pool_size="15GB", CUDA_VISIBLE_DEVICES="1,2,3" + ) client = Client(cluster) rmm.reinitialize(pool_allocator=True) @@ -74,7 +88,6 @@ def run_bandwidth_test(ddf, n): run_bandwidth_test(ddf, 4_000_000) run_bandwidth_test(ddf, -1) - print("--"*20+"Completed Test"+"--"*20, flush=True) + print("--" * 20 + "Completed Test" + "--" * 20, flush=True) client.shutdown() cluster.close() - diff --git a/benchmarks/shared/build_cugraph_ucx/test_cugraph_sampling.py b/benchmarks/shared/build_cugraph_ucx/test_cugraph_sampling.py index 110ad80838a..2903e1607df 100644 --- a/benchmarks/shared/build_cugraph_ucx/test_cugraph_sampling.py +++ b/benchmarks/shared/build_cugraph_ucx/test_cugraph_sampling.py @@ -23,19 +23,22 @@ _seed = 42 + def benchmark_func(func, n_times=10): def wrap_func(*args, **kwargs): time_ls = [] # ignore 1st run # and return other runs - for _ in range(0,n_times+1): + for _ in range(0, n_times + 1): t1 = perf_counter_ns() result = func(*args, **kwargs) t2 = perf_counter_ns() - time_ls.append(t2-t1) + time_ls.append(t2 - t1) return result, time_ls[1:] + return wrap_func + def create_mg_graph(graph_data): """ Create a graph instance based on the data to be loaded/generated. @@ -67,41 +70,47 @@ def create_mg_graph(graph_data): ) return G + @benchmark_func def sample_graph(G, start_list): - output_ddf = uniform_neighbor_sample_mg(G,start_list=start_list, fanout_vals=[10,25]) + output_ddf = uniform_neighbor_sample_mg( + G, start_list=start_list, fanout_vals=[10, 25] + ) df = output_ddf.compute() return df + def run_sampling_test(ddf, start_list): df, time_ls = sample_graph(ddf, start_list) time_ar = np.asarray(time_ls) time_mean = time_ar.mean() print(f"Sampling {len(start_list):,} took = {time_mean*1e-6} ms") return - if __name__ == "__main__": - cluster = LocalCUDACluster(protocol='ucx',rmm_pool_size='15GB', CUDA_VISIBLE_DEVICES='1,2,3,4,5,6,7,8') + cluster = LocalCUDACluster( + protocol="ucx", rmm_pool_size="15GB", CUDA_VISIBLE_DEVICES="1,2,3,4,5,6,7,8" + ) client = Client(cluster) Comms.initialize(p2p=True) rmm.reinitialize(pool_allocator=True) - graph_data = {"scale": 26, - "edgefactor": 8 , - } - + graph_data = { + "scale": 26, + "edgefactor": 8, + } + g = create_mg_graph(graph_data) for num_start_verts in [1_000, 10_000, 100_000]: start_list = g.input_df["src"].head(num_start_verts) - assert len(start_list)==num_start_verts + assert len(start_list) == num_start_verts run_sampling_test(g, start_list) - - print("--"*20+"Completed Test"+"--"*20, flush=True) + + print("--" * 20 + "Completed Test" + "--" * 20, flush=True) Comms.destroy() client.shutdown() - cluster.close() \ No newline at end of file + cluster.close() diff --git a/benchmarks/shared/python/cugraph_benchmarking/params.py b/benchmarks/shared/python/cugraph_benchmarking/params.py index ee63b8768a6..d82cfd26117 100644 --- a/benchmarks/shared/python/cugraph_benchmarking/params.py +++ b/benchmarks/shared/python/cugraph_benchmarking/params.py @@ -15,9 +15,11 @@ from pylibcugraph.testing.utils import gen_fixture_params from cugraph.testing import RAPIDS_DATASET_ROOT_DIR_PATH -from cugraph.experimental.datasets import ( +from cugraph.datasets import ( Dataset, karate, + netscience, + email_Eu_core, ) # Create Dataset objects from .csv files. @@ -26,67 +28,104 @@ hollywood = Dataset( csv_file=RAPIDS_DATASET_ROOT_DIR_PATH / "csv/undirected/hollywood.csv", csv_col_names=["src", "dst"], - csv_col_dtypes=["int32", "int32"]) + csv_col_dtypes=["int32", "int32"], +) +hollywood.metadata["is_directed"] = False europe_osm = Dataset( csv_file=RAPIDS_DATASET_ROOT_DIR_PATH / "csv/undirected/europe_osm.csv", csv_col_names=["src", "dst"], - csv_col_dtypes=["int32", "int32"]) + csv_col_dtypes=["int32", "int32"], +) +europe_osm.metadata["is_directed"] = False cit_patents = Dataset( csv_file=RAPIDS_DATASET_ROOT_DIR_PATH / "csv/directed/cit-Patents.csv", csv_col_names=["src", "dst"], - csv_col_dtypes=["int32", "int32"]) + csv_col_dtypes=["int32", "int32"], +) +cit_patents.metadata["is_directed"] = True soc_livejournal = Dataset( csv_file=RAPIDS_DATASET_ROOT_DIR_PATH / "csv/directed/soc-LiveJournal1.csv", csv_col_names=["src", "dst"], - csv_col_dtypes=["int32", "int32"]) + csv_col_dtypes=["int32", "int32"], +) +soc_livejournal.metadata["is_directed"] = True # Assume all "file_data" (.csv file on disk) datasets are too small to be useful for MG. undirected_datasets = [ - pytest.param(karate, - marks=[pytest.mark.tiny, - pytest.mark.undirected, - pytest.mark.file_data, - pytest.mark.sg, - ]), - pytest.param(hollywood, - marks=[pytest.mark.small, - pytest.mark.undirected, - pytest.mark.file_data, - pytest.mark.sg, - ]), - pytest.param(europe_osm, - marks=[pytest.mark.undirected, - pytest.mark.file_data, - pytest.mark.sg, - ]), + pytest.param( + karate, + marks=[ + pytest.mark.tiny, + pytest.mark.undirected, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), + pytest.param( + hollywood, + marks=[ + pytest.mark.small, + pytest.mark.undirected, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), + pytest.param( + europe_osm, + marks=[ + pytest.mark.undirected, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), ] directed_datasets = [ - pytest.param(cit_patents, - marks=[pytest.mark.small, - pytest.mark.directed, - pytest.mark.file_data, - pytest.mark.sg, - ]), - pytest.param(soc_livejournal, - marks=[pytest.mark.directed, - pytest.mark.file_data, - pytest.mark.sg, - ]), + pytest.param( + netscience, + marks=[ + pytest.mark.small, + pytest.mark.directed, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), + pytest.param( + email_Eu_core, + marks=[ + pytest.mark.small, + pytest.mark.directed, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), + pytest.param( + cit_patents, + marks=[ + pytest.mark.small, + pytest.mark.directed, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), + pytest.param( + soc_livejournal, + marks=[ + pytest.mark.directed, + pytest.mark.file_data, + pytest.mark.sg, + ], + ), ] managed_memory = [ - pytest.param(True, - marks=[pytest.mark.managedmem_on]), - pytest.param(False, - marks=[pytest.mark.managedmem_off]), + pytest.param(True, marks=[pytest.mark.managedmem_on]), + pytest.param(False, marks=[pytest.mark.managedmem_off]), ] pool_allocator = [ - pytest.param(True, - marks=[pytest.mark.poolallocator_on]), - pytest.param(False, - marks=[pytest.mark.poolallocator_off]), + pytest.param(True, marks=[pytest.mark.poolallocator_on]), + pytest.param(False, marks=[pytest.mark.poolallocator_off]), ] sg = pytest.param( @@ -117,17 +156,31 @@ for scale in _rmat_scales: for edgefactor in _rmat_edgefactors: rmat[f"{scale}_{edgefactor}"] = pytest.param( - {"scale": scale, - "edgefactor": edgefactor, - }, + { + "scale": scale, + "edgefactor": edgefactor, + }, id=f"dataset=rmat_{scale}_{edgefactor}", ) # sampling algos length of start list -_batch_sizes = [100, 500, 1000, 2500, 5000, - 10000, 20000, 30000, 40000, - 50000, 60000, 70000, 80000, - 90000, 100000] +_batch_sizes = [ + 100, + 500, + 1000, + 2500, + 5000, + 10000, + 20000, + 30000, + 40000, + 50000, + 60000, + 70000, + 80000, + 90000, + 100000, +] batch_sizes = {} for bs in _batch_sizes: batch_sizes[bs] = pytest.param( @@ -152,11 +205,10 @@ _num_clients = [2, 4, 8, 16, 32] num_clients = {} for nc in _num_clients: - num_clients[nc] = ( - pytest.param(nc, - id=f"num_clients={nc}", - marks=[getattr(pytest.mark, f"num_clients_{nc}")], - ) + num_clients[nc] = pytest.param( + nc, + id=f"num_clients={nc}", + marks=[getattr(pytest.mark, f"num_clients_{nc}")], ) # Parameters for Graph generation fixture diff --git a/build.sh b/build.sh index 8dca89aeedd..eef19046d85 100755 --- a/build.sh +++ b/build.sh @@ -18,20 +18,24 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) +RAPIDS_VERSION=23.12 + # Valid args to this script (all possible targets and options) - only one per line VALIDARGS=" clean uninstall libcugraph libcugraph_etl + pylibcugraph cugraph cugraph-service - pylibcugraph - cpp-mgtests cugraph-pyg cugraph-dgl nx-cugraph + cpp-mgtests + cpp-mtmgtests docs + all -v -g -n @@ -52,13 +56,15 @@ HELP="$0 [ ...] [ ...] libcugraph - build libcugraph.so and SG test binaries libcugraph_etl - build libcugraph_etl.so and SG test binaries pylibcugraph - build the pylibcugraph Python package - cugraph-pyg - build the cugraph-pyg Python package cugraph - build the cugraph Python package - nx-cugraph - build the nx-cugraph Python package cugraph-service - build the cugraph-service_client and cugraph-service_server Python package - cpp-mgtests - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency. + cugraph-pyg - build the cugraph-pyg Python package cugraph-dgl - build the cugraph-dgl extensions for DGL + nx-cugraph - build the nx-cugraph Python package + cpp-mgtests - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency. + cpp-mtmgtests - build libcugraph MTMG tests. Adds UCX as a dependency (temporary). docs - build the docs + all - build everything and is: -v - verbose build mode -g - build for debug @@ -71,7 +77,7 @@ HELP="$0 [ ...] [ ...] --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) -h - print this text - default action (no args) is to build and install 'libcugraph' then 'libcugraph_etl' then 'pylibcugraph' then 'cugraph' targets + default action (no args) is to build and install 'libcugraph' then 'libcugraph_etl' then 'pylibcugraph' and then 'cugraph' targets libcugraph build dir is: ${LIBCUGRAPH_BUILD_DIR} @@ -103,6 +109,7 @@ BUILD_TYPE=Release INSTALL_TARGET="--target install" BUILD_CPP_TESTS=ON BUILD_CPP_MG_TESTS=OFF +BUILD_CPP_MTMG_TESTS=OFF BUILD_ALL_GPU_ARCH=0 BUILD_WITH_CUGRAPHOPS=ON CMAKE_GENERATOR_OPTION="-G Ninja" @@ -119,7 +126,7 @@ function hasArg { (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } -function buildAll { +function buildDefault { (( ${NUMARGS} == 0 )) || !(echo " ${ARGS} " | grep -q " [^-][a-zA-Z0-9\_\-]\+ ") } @@ -170,7 +177,10 @@ fi if hasArg --without_cugraphops; then BUILD_WITH_CUGRAPHOPS=OFF fi -if hasArg cpp-mgtests; then +if hasArg cpp-mtmgtests; then + BUILD_CPP_MTMG_TESTS=ON +fi +if hasArg cpp-mgtests || hasArg all; then BUILD_CPP_MG_TESTS=ON fi if hasArg --cmake_default_generator; then @@ -240,7 +250,7 @@ fi ################################################################################ # Configure, build, and install libcugraph -if buildAll || hasArg libcugraph; then +if buildDefault || hasArg libcugraph || hasArg all; then if hasArg --clean; then if [ -d ${LIBCUGRAPH_BUILD_DIR} ]; then find ${LIBCUGRAPH_BUILD_DIR} -mindepth 1 -delete @@ -262,6 +272,7 @@ if buildAll || hasArg libcugraph; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_TESTS=${BUILD_CPP_TESTS} \ -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \ + -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \ -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \ ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} @@ -270,7 +281,7 @@ if buildAll || hasArg libcugraph; then fi # Configure, build, and install libcugraph_etl -if buildAll || hasArg libcugraph_etl; then +if buildDefault || hasArg libcugraph_etl || hasArg all; then if hasArg --clean; then if [ -d ${LIBCUGRAPH_ETL_BUILD_DIR} ]; then find ${LIBCUGRAPH_ETL_BUILD_DIR} -mindepth 1 -delete @@ -292,6 +303,7 @@ if buildAll || hasArg libcugraph_etl; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_TESTS=${BUILD_CPP_TESTS} \ -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \ + -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \ -DCMAKE_PREFIX_PATH=${LIBCUGRAPH_BUILD_DIR} \ ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} \ @@ -301,7 +313,7 @@ if buildAll || hasArg libcugraph_etl; then fi # Build, and install pylibcugraph -if buildAll || hasArg pylibcugraph; then +if buildDefault || hasArg pylibcugraph || hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/pylibcugraph else @@ -328,7 +340,7 @@ if buildAll || hasArg pylibcugraph; then fi # Build and install the cugraph Python package -if buildAll || hasArg cugraph; then +if buildDefault || hasArg cugraph || hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/cugraph else @@ -355,7 +367,7 @@ if buildAll || hasArg cugraph; then fi # Install the cugraph-service-client and cugraph-service-server Python packages -if hasArg cugraph-service; then +if hasArg cugraph-service || hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/cugraph-service else @@ -365,7 +377,7 @@ if hasArg cugraph-service; then fi # Build and install the cugraph-pyg Python package -if hasArg cugraph-pyg; then +if hasArg cugraph-pyg || hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/cugraph-pyg else @@ -374,7 +386,7 @@ if hasArg cugraph-pyg; then fi # Install the cugraph-dgl extensions for DGL -if hasArg cugraph-dgl; then +if hasArg cugraph-dgl || hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/cugraph-dgl else @@ -383,7 +395,7 @@ if hasArg cugraph-dgl; then fi # Build and install the nx-cugraph Python package -if hasArg nx-cugraph; then +if hasArg nx-cugraph || hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/nx-cugraph else @@ -392,7 +404,7 @@ if hasArg nx-cugraph; then fi # Build the docs -if hasArg docs; then +if hasArg docs || hasArg all; then if [ ! -d ${LIBCUGRAPH_BUILD_DIR} ]; then mkdir -p ${LIBCUGRAPH_BUILD_DIR} cd ${LIBCUGRAPH_BUILD_DIR} @@ -402,8 +414,28 @@ if hasArg docs; then ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} fi + + for PROJECT in libcugraphops libwholegraph; do + XML_DIR="${REPODIR}/docs/cugraph/${PROJECT}" + rm -rf "${XML_DIR}" + mkdir -p "${XML_DIR}" + export XML_DIR_${PROJECT^^}="$XML_DIR" + + echo "downloading xml for ${PROJECT} into ${XML_DIR}. Environment variable XML_DIR_${PROJECT^^} is set to ${XML_DIR}" + curl -O "https://d1664dvumjb44w.cloudfront.net/${PROJECT}/xml_tar/${RAPIDS_VERSION}/xml.tar.gz" + tar -xzf xml.tar.gz -C "${XML_DIR}" + rm "./xml.tar.gz" + done + cd ${LIBCUGRAPH_BUILD_DIR} cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} --target docs_cugraph ${VERBOSE_FLAG} + + echo "making libcugraph doc dir" + rm -rf ${REPODIR}/docs/cugraph/libcugraph + mkdir -p ${REPODIR}/docs/cugraph/libcugraph + + export XML_DIR_LIBCUGRAPH="${REPODIR}/cpp/doxygen/xml" + cd ${REPODIR}/docs/cugraph make html fi diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 3fb72cac08b..d0d13f99448 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,8 +9,10 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +version=$(rapids-generate-version) + rapids-logger "Begin cpp build" -rapids-conda-retry mambabuild conda/recipes/libcugraph +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 2941d062d80..3f765704bdb 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -29,7 +29,9 @@ rapids-mamba-retry install \ cugraph-pyg \ cugraph-service-server \ cugraph-service-client \ - libcugraph_etl + libcugraph_etl \ + pylibcugraphops \ + pylibwholegraph # This command installs `cugraph-dgl` without its dependencies # since this package can currently only run in `11.6` CTK environments @@ -37,14 +39,20 @@ rapids-mamba-retry install \ rapids-logger "Install cugraph-dgl" rapids-mamba-retry install "${PYTHON_CHANNEL}/linux-64/cugraph-dgl-*.tar.bz2" -export RAPIDS_VERSION_NUMBER="23.10" +export RAPIDS_VERSION_NUMBER="23.12" export RAPIDS_DOCS_DIR="$(mktemp -d)" +for PROJECT in libcugraphops libwholegraph; do + rapids-logger "Download ${PROJECT} xml_tar" + TMP_DIR=$(mktemp -d) + export XML_DIR_${PROJECT^^}="$TMP_DIR" + curl "https://d1664dvumjb44w.cloudfront.net/${PROJECT}/xml_tar/${RAPIDS_VERSION_NUMBER}/xml.tar.gz" | tar -xzf - -C "${TMP_DIR}" +done + rapids-logger "Build CPP docs" pushd cpp/doxygen doxygen Doxyfile -mkdir -p "${RAPIDS_DOCS_DIR}/libcugraph/html" -mv html/* "${RAPIDS_DOCS_DIR}/libcugraph/html" +export XML_DIR_LIBCUGRAPH="$(pwd)/xml" popd rapids-logger "Build Python docs" diff --git a/ci/build_python.sh b/ci/build_python.sh index 62eb6c2ccec..90a40c539ff 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,8 +11,19 @@ rapids-print-env CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) +export RAPIDS_PACKAGE_VERSION=${version} +echo "${version}" > VERSION + rapids-logger "Begin py build" +package_dir="python" +for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do + underscore_package_name=$(echo "${package_name}" | tr "-" "_") + sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" +done + # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ @@ -40,6 +51,10 @@ rapids-conda-retry mambabuild \ # built on each CUDA platform to ensure they are included in each set of # artifacts, since test scripts only install from one set of artifacts based on # the CUDA version used for the test run. +version_file_cugraph_service_client="python/cugraph-service/client/cugraph_service_client/_version.py" +sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_client} +version_file_cugraph_service_server="python/cugraph-service/server/cugraph_service_server/_version.py" +sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_server} rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 821aa25c1b9..163520ea1da 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,13 +5,13 @@ set -euo pipefail package_name=$1 package_dir=$2 +underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string -# Use gha-tools rapids-pip-wheel-version to generate wheel version then -# update the necessary files -version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" @@ -21,9 +21,11 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" # Patch project metadata files to include the CUDA version suffix and version override. pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${underscore_package_name}/_version.py" -sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} # For nightlies we want to ensure that we're pulling in alphas as well. The # easiest way to do so is to augment the spec with a constraint containing a @@ -38,8 +40,11 @@ for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done -# dask-cuda doesn't get a suffix, but it does get an alpha spec. -sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} +# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. +for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} +done + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index aaeaa715434..c091bd1ed33 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -54,24 +54,14 @@ sed_runner "s/set(cugraph_version .*)/set(cugraph_version ${NEXT_FULL_TAG})/g" p sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/cugraph/source/conf.py sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cugraph/source/conf.py -# Python __init__.py updates -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph/cugraph/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-dgl/cugraph_dgl/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-pyg/cugraph_pyg/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/client/cugraph_service_client/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/server/cugraph_service_server/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/pylibcugraph/pylibcugraph/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/nx_cugraph/__init__.py -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/_nx_cugraph/__init__.py -# Python pyproject.toml updates -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph/pyproject.toml -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-dgl/pyproject.toml -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-pyg/pyproject.toml -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/client/pyproject.toml -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/server/pyproject.toml -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/pylibcugraph/pyproject.toml -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/pyproject.toml +# build.sh script +sed_runner 's/RAPIDS_VERSION=.*/RAPIDS_VERSION='${NEXT_SHORT_TAG}'/g' build.sh + +# Centralized version file update +# NOTE: Any script that runs in CI will need to use gha-tool `rapids-generate-version` +# and echo it to `VERSION` file to get an alpha spec of the current version +echo "${NEXT_FULL_TAG}" > VERSION # Wheel testing script sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cugraph.sh @@ -102,10 +92,12 @@ DEPENDENCIES=( raft-dask rmm ucx-py + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do - for FILE in dependencies.yaml conda/environments/*.yaml; do + for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} + sed_runner "/-.* ${DEP}-cu[0-9][0-9]==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" ${FILE} done for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do @@ -122,9 +114,14 @@ sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" cond sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/cugraph-service/conda_build_config.yaml sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/pylibcugraph/conda_build_config.yaml +# nx-cugraph NetworkX entry-point meta-data +sed_runner "s@branch-[0-9][0-9].[0-9][0-9]@branch-${NEXT_SHORT_TAG}@g" python/nx-cugraph/_nx_cugraph/__init__.py +# FIXME: can this use the standard VERSION file and update mechanism? +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/_nx_cugraph/__init__.py + # CI files for FILE in .github/workflows/*.yaml; do - sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" + sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" # Wheel builds clone cugraph-ops, update its branch sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" "${FILE}" # Wheel builds install dask-cuda from source, update its branch diff --git a/ci/test_python.sh b/ci/test_python.sh index df0f34377a3..d6e92e8d1a5 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -63,12 +63,13 @@ pytest \ tests popd +# FIXME: TEMPORARILY disable single-GPU "MG" testing rapids-logger "pytest cugraph" pushd python/cugraph/cugraph DASK_WORKER_DEVICES="0" \ DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \ DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \ -DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT=20 \ +DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \ pytest \ -v \ --benchmark-disable \ @@ -196,27 +197,26 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then conda activate test_cugraph_pyg set -u - # Install pytorch + # Will automatically install built dependencies of cuGraph-PyG rapids-mamba-retry install \ - --force-reinstall \ - --channel pyg \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ --channel pytorch \ --channel nvidia \ - 'pyg=2.3' \ - 'pytorch=2.0.0' \ - 'pytorch-cuda=11.8' + --channel pyg \ + --channel rapidsai-nightly \ + "cugraph-pyg" \ + "pytorch>=2.0,<2.1" \ + "pytorch-cuda=11.8" # Install pyg dependencies (which requires pip) - pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cu118.html - - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - libcugraph \ - pylibcugraph \ - pylibcugraphops \ - cugraph \ - cugraph-pyg + pip install \ + pyg_lib \ + torch_scatter \ + torch_sparse \ + torch_cluster \ + torch_spline_conv \ + -f https://data.pyg.org/whl/torch-2.0.0+cu118.html rapids-print-env diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index d6ec67cd9e9..428efd4ed21 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -21,9 +21,10 @@ arch=$(uname -m) if [[ "${arch}" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test_${package_name}.py else + # FIXME: TEMPORARILY disable single-GPU "MG" testing RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets \ DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \ DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \ - DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT=20 \ + DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \ python -m pytest ./python/${package_name}/${python_package_name}/tests fi diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh index ac18459128a..d351ea21624 100755 --- a/ci/test_wheel_cugraph.sh +++ b/ci/test_wheel_cugraph.sh @@ -8,7 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl -# Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@2023.9.2 git+https://github.com/dask/distributed.git@2023.9.2 - ./ci/test_wheel.sh cugraph python/cugraph diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index dea52887f23..aa38defcd7c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -11,19 +11,17 @@ channels: - nvidia dependencies: - aiohttp +- breathe - c-compiler - cmake>=3.26.4 - cuda-version=11.8 - cudatoolkit -- cudf==23.10.* +- cudf==23.12.* - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core==2023.9.2 -- dask-cuda==23.10.* -- dask-cudf==23.10.* -- dask==2023.9.2 -- distributed==2023.9.2 +- dask-cuda==23.12.* +- dask-cudf==23.12.* - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* @@ -31,11 +29,11 @@ dependencies: - graphviz - gtest>=1.13.0 - ipython -- libcudf==23.10.* -- libcugraphops==23.10.* -- libraft-headers==23.10.* -- libraft==23.10.* -- librmm==23.10.* +- libcudf==23.12.* +- libcugraphops==23.12.* +- libraft-headers==23.12.* +- libraft==23.12.* +- librmm==23.12.* - nbsphinx - nccl>=2.9.9 - networkx>=2.5.1 @@ -51,19 +49,20 @@ dependencies: - pandas - pre-commit - pydata-sphinx-theme -- pylibcugraphops==23.10.* -- pylibraft==23.10.* -- pylibwholegraph==23.10.* +- pylibcugraphops==23.12.* +- pylibraft==23.12.* +- pylibwholegraph==23.12.* - pytest - pytest-benchmark - pytest-cov - pytest-mpl - pytest-xdist - python-louvain -- raft-dask==23.10.* +- raft-dask==23.12.* +- rapids-dask-dependency==23.12.* - recommonmark - requests -- rmm==23.10.* +- rmm==23.12.* - scikit-build>=0.13.1 - scikit-learn>=0.23.1 - scipy @@ -73,7 +72,7 @@ dependencies: - sphinx<6 - sphinxcontrib-websupport - ucx-proc=*=gpu -- ucx-py==0.34.* +- ucx-py==0.35.* - wget - wheel name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 2d55f73c5d1..a9f793b15f5 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -11,19 +11,17 @@ channels: - nvidia dependencies: - aiohttp +- breathe - c-compiler - cmake>=3.26.4 - cuda-nvcc - cuda-version=12.0 -- cudf==23.10.* +- cudf==23.12.* - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core==2023.9.2 -- dask-cuda==23.10.* -- dask-cudf==23.10.* -- dask==2023.9.2 -- distributed==2023.9.2 +- dask-cuda==23.12.* +- dask-cudf==23.12.* - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* @@ -31,11 +29,11 @@ dependencies: - graphviz - gtest>=1.13.0 - ipython -- libcudf==23.10.* -- libcugraphops==23.10.* -- libraft-headers==23.10.* -- libraft==23.10.* -- librmm==23.10.* +- libcudf==23.12.* +- libcugraphops==23.12.* +- libraft-headers==23.12.* +- libraft==23.12.* +- librmm==23.12.* - nbsphinx - nccl>=2.9.9 - networkx>=2.5.1 @@ -50,19 +48,20 @@ dependencies: - pandas - pre-commit - pydata-sphinx-theme -- pylibcugraphops==23.10.* -- pylibraft==23.10.* -- pylibwholegraph==23.10.* +- pylibcugraphops==23.12.* +- pylibraft==23.12.* +- pylibwholegraph==23.12.* - pytest - pytest-benchmark - pytest-cov - pytest-mpl - pytest-xdist - python-louvain -- raft-dask==23.10.* +- raft-dask==23.12.* +- rapids-dask-dependency==23.12.* - recommonmark - requests -- rmm==23.10.* +- rmm==23.12.* - scikit-build>=0.13.1 - scikit-learn>=0.23.1 - scipy @@ -72,7 +71,7 @@ dependencies: - sphinx<6 - sphinxcontrib-websupport - ucx-proc=*=gpu -- ucx-py==0.34.* +- ucx-py==0.35.* - wget - wheel name: all_cuda-120_arch-x86_64 diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml index 9e9fcd2faf1..aaa1cd8a936 100644 --- a/conda/recipes/cugraph-dgl/meta.yaml +++ b/conda/recipes/cugraph-dgl/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -10,7 +10,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} @@ -26,7 +26,7 @@ requirements: - dgl >=1.1.0.cu* - numba >=0.57 - numpy >=1.21 - - pylibcugraphops ={{ version }} + - pylibcugraphops ={{ minor_version }} - python - pytorch diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 1dc5a75c41b..a2a02a1d9f6 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -10,7 +10,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} @@ -26,15 +26,15 @@ requirements: - python - scikit-build >=0.13.1 run: - - distributed ==2023.9.2 + - rapids-dask-dependency ={{ minor_version }} - numba >=0.57 - numpy >=1.21 - python - pytorch >=2.0 - cupy >=12.0.0 - cugraph ={{ version }} - - pylibcugraphops ={{ version }} - - pyg >=2.3,<2.4 + - pylibcugraphops ={{ minor_version }} + - pyg >=2.3,<2.5 tests: imports: diff --git a/conda/recipes/cugraph-service/conda_build_config.yaml b/conda/recipes/cugraph-service/conda_build_config.yaml index 5fe8d372eba..b971a73fd39 100644 --- a/conda/recipes/cugraph-service/conda_build_config.yaml +++ b/conda/recipes/cugraph-service/conda_build_config.yaml @@ -1,2 +1,2 @@ ucx_py_version: - - "0.34.*" + - "0.35.*" diff --git a/conda/recipes/cugraph-service/meta.yaml b/conda/recipes/cugraph-service/meta.yaml index 2daf0438351..d52a004db05 100644 --- a/conda/recipes/cugraph-service/meta.yaml +++ b/conda/recipes/cugraph-service/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2018-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -9,7 +9,7 @@ package: name: cugraph-service-split source: - git_url: ../../.. + path: ../../.. outputs: - name: cugraph-service-client @@ -59,10 +59,10 @@ outputs: - cupy >=12.0.0 - dask-cuda ={{ minor_version }} - dask-cudf ={{ minor_version }} - - distributed ==2023.9.2 - numba >=0.57 - numpy >=1.21 - python + - rapids-dask-dependency ={{ minor_version }} - thriftpy2 >=0.4.15 - ucx-py {{ ucx_py_version }} diff --git a/conda/recipes/cugraph/conda_build_config.yaml b/conda/recipes/cugraph/conda_build_config.yaml index ba5b46dda1f..c03d515b9f6 100644 --- a/conda/recipes/cugraph/conda_build_config.yaml +++ b/conda/recipes/cugraph/conda_build_config.yaml @@ -17,4 +17,4 @@ sysroot_version: - "2.17" ucx_py_version: - - "0.34.*" + - "0.35.*" diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml index f9bf54a2ef4..58b9ea220d4 100644 --- a/conda/recipes/cugraph/meta.yaml +++ b/conda/recipes/cugraph/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2018-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} @@ -12,7 +12,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} @@ -76,15 +76,13 @@ requirements: - cupy >=12.0.0 - dask-cuda ={{ minor_version }} - dask-cudf ={{ minor_version }} - - dask ==2023.9.2 - - dask-core ==2023.9.2 - - distributed ==2023.9.2 - fsspec>=0.6.0 - libcugraph ={{ version }} - pylibcugraph ={{ version }} - pylibraft ={{ minor_version }} - python - raft-dask ={{ minor_version }} + - rapids-dask-dependency ={{ minor_version }} - requests - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 83c82adf703..66f72e6b6b5 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2018-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} @@ -10,7 +10,7 @@ package: name: libcugraph-split source: - git_url: ../../.. + path: ../../.. build: script_env: diff --git a/conda/recipes/nx-cugraph/meta.yaml b/conda/recipes/nx-cugraph/meta.yaml index 556d72e8548..cdb7bc13c23 100644 --- a/conda/recipes/nx-cugraph/meta.yaml +++ b/conda/recipes/nx-cugraph/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -10,7 +10,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/conda/recipes/pylibcugraph/conda_build_config.yaml b/conda/recipes/pylibcugraph/conda_build_config.yaml index ba5b46dda1f..c03d515b9f6 100644 --- a/conda/recipes/pylibcugraph/conda_build_config.yaml +++ b/conda/recipes/pylibcugraph/conda_build_config.yaml @@ -17,4 +17,4 @@ sysroot_version: - "2.17" ucx_py_version: - - "0.34.*" + - "0.35.*" diff --git a/conda/recipes/pylibcugraph/meta.yaml b/conda/recipes/pylibcugraph/meta.yaml index 083998be053..ad59c4de66f 100644 --- a/conda/recipes/pylibcugraph/meta.yaml +++ b/conda/recipes/pylibcugraph/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} @@ -12,7 +12,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d9f87d7dd72..836d5569ef7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -25,7 +25,7 @@ include(rapids-find) rapids_cuda_init_architectures(CUGRAPH) -project(CUGRAPH VERSION 23.10.00 LANGUAGES C CXX CUDA) +project(CUGRAPH VERSION 23.12.00 LANGUAGES C CXX CUDA) if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0) @@ -153,6 +153,11 @@ rapids_cpm_init() # lags behind. ### +# Need to make sure rmm is found before cuco so that rmm patches the libcudacxx +# directory to be found by cuco. +include(${rapids-cmake-dir}/cpm/rmm.cmake) +rapids_cpm_rmm(BUILD_EXPORT_SET cugraph-exports + INSTALL_EXPORT_SET cugraph-exports) # Putting this before raft to override RAFT from pulling them in. include(cmake/thirdparty/get_libcudacxx.cmake) include(${rapids-cmake-dir}/cpm/cuco.cmake) @@ -166,7 +171,10 @@ endif() include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_cuhornet.cmake) -include(cmake/thirdparty/get_ucp.cmake) + +if (BUILD_CUGRAPH_MTMG_TESTS) + include(cmake/thirdparty/get_ucp.cmake) +endif() if(BUILD_TESTS) include(cmake/thirdparty/get_gtest.cmake) @@ -184,6 +192,7 @@ set(CUGRAPH_SOURCES src/detail/shuffle_vertex_pairs.cu src/detail/collect_local_vertex_values.cu src/detail/groupby_and_count.cu + src/detail/collect_comm_wrapper.cu src/sampling/random_walks_mg.cu src/community/detail/common_methods_mg.cu src/community/detail/common_methods_sg.cu @@ -193,6 +202,8 @@ set(CUGRAPH_SOURCES src/community/detail/mis_mg.cu src/detail/utility_wrappers.cu src/structure/graph_view_mg.cu + src/structure/remove_self_loops.cu + src/structure/remove_multi_edges.cu src/utilities/path_retrieval.cu src/structure/legacy/graph.cu src/linear_assignment/legacy/hungarian.cu @@ -435,6 +446,7 @@ add_library(cugraph_c src/c_api/labeling_result.cpp src/c_api/weakly_connected_components.cpp src/c_api/strongly_connected_components.cpp + src/c_api/allgather.cpp src/c_api/legacy_k_truss.cpp ) add_library(cugraph::cugraph_c ALIAS cugraph_c) diff --git a/cpp/doxygen/Doxyfile b/cpp/doxygen/Doxyfile index eb414925388..6946bd38bfe 100644 --- a/cpp/doxygen/Doxyfile +++ b/cpp/doxygen/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.8.20 +# Doxyfile 1.9.8 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -12,6 +12,16 @@ # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables or CMake type +# replacement variables: +# doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options @@ -32,19 +42,19 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "libcugraph" +PROJECT_NAME = libcugraph # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER=23.10 +PROJECT_NUMBER = 23.12 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. -PROJECT_BRIEF = GPU accelerated graph analytics +PROJECT_BRIEF = "GPU accelerated graph analytics" # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 @@ -60,16 +70,28 @@ PROJECT_LOGO = OUTPUT_DIRECTORY = -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# number of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode @@ -81,26 +103,18 @@ ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -248,16 +262,16 @@ TAB_SIZE = 4 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -302,8 +316,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -313,7 +327,10 @@ OPTIMIZE_OUTPUT_SLICE = NO # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = cu=C++ \ cuh=C++ @@ -337,6 +354,17 @@ MARKDOWN_SUPPORT = YES TOC_INCLUDE_HEADINGS = 5 +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0 and GITHUB use the lower case version of title +# with any whitespace replaced by '-' and punctuation characters removed. +# The default value is: DOXYGEN. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +MARKDOWN_ID_STYLE = DOXYGEN + # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or @@ -448,19 +476,27 @@ TYPEDEF_HIDES_STRUCT = NO LOOKUP_CACHE_SIZE = 0 -# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use # during processing. When set to 0 doxygen will based this on the number of # cores available in the system. You can set it explicitly to a value larger # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. NUM_PROC_THREADS = 1 +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = NO + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -524,6 +560,13 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation @@ -535,7 +578,8 @@ HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. +# will also hide undocumented C++ concepts if enabled. This option has no effect +# if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO @@ -561,12 +605,20 @@ HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# (including Cygwin) and Mac users are advised to set this option to NO. -# The default value is: system dependent. +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# Possible values are: SYSTEM, NO and YES. +# The default value is: SYSTEM. CASE_SENSE_NAMES = YES @@ -584,6 +636,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -741,7 +799,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -787,24 +846,50 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES +# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, doxygen will warn about +# undocumented enumeration values. If set to NO, doxygen will accept +# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: NO. + +WARN_IF_UNDOC_ENUM_VAL = NO + # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO @@ -815,13 +900,27 @@ WARN_AS_ERROR = NO # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -842,12 +941,23 @@ INPUT = main_page.md \ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 +# This tag can be used to specify the character encoding of the source files +# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify +# character encoding on a per file pattern basis. Doxygen will compare the file +# name with each pattern and apply the encoding instead of the default +# INPUT_ENCODING) if there is a match. The character encodings are a list of the +# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding +# "INPUT_ENCODING" for further information on supported encodings. + +INPUT_FILE_ENCODING = + # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. @@ -856,13 +966,15 @@ INPUT_ENCODING = UTF-8 # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen -# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, -# *.vhdl, *.ucf, *.qsf and *.ice. +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cxxm, +# *.cpp, *.cppm, *.c++, *.c++m, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, +# *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, *.h++, *.ixx, *.l, *.cs, *.d, *.php, +# *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be +# provided as doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.cpp \ *.hpp \ @@ -907,10 +1019,7 @@ EXCLUDE_PATTERNS = */nvtx/* \ # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* +# ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = org::apache @@ -955,6 +1064,11 @@ IMAGE_PATH = # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # +# Note that doxygen will use the data processed and written to standard output +# for further processing, therefore nothing else, like debug statements or used +# commands (so in case of a Windows batch file always use @echo OFF), should be +# written to standard output. +# # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. @@ -996,6 +1110,15 @@ FILTER_SOURCE_PATTERNS = USE_MDFILE_AS_MAINPAGE = main_page.md +# The Fortran standard specifies that for fixed formatted Fortran code all +# characters from position 72 are to be considered as comment. A common +# extension is to allow longer lines before the automatic comment starts. The +# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can +# be processed before the automatic comment starts. +# Minimum value: 7, maximum value: 10000, default value: 72. + +FORTRAN_COMMENT_AFTER = 72 + #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- @@ -1093,17 +1216,11 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. +# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) +# that should be ignored while generating the index headers. The IGNORE_PREFIX +# tag works for classes, function and member names. The entity will be placed in +# the alphabetical list under the first letter of the entity name that remains +# after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = @@ -1115,7 +1232,7 @@ IGNORE_PREFIX = # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output # The default value is: YES. -GENERATE_HTML = YES +GENERATE_HTML = NO # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1182,7 +1299,12 @@ HTML_STYLESHEET = # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. +# list). +# Note: Since the styling of scrollbars can currently not be overruled in +# Webkit/Chromium, the styling will be left out of the default doxygen.css if +# one or more extra stylesheets have been specified. So if scrollbar +# customization is desired it has to be added explicitly. For an example see the +# documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = @@ -1197,9 +1319,22 @@ HTML_EXTRA_STYLESHEET = HTML_EXTRA_FILES = +# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output +# should be rendered with a dark or light theme. +# Possible values are: LIGHT always generate light mode output, DARK always +# generate dark mode output, AUTO_LIGHT automatically set the mode according to +# the user preference, use light mode if no preference is set (the default), +# AUTO_DARK automatically set the mode according to the user preference, use +# dark mode if no preference is set and TOGGLE allow to user to switch between +# light and dark mode via a button. +# The default value is: AUTO_LIGHT. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE = AUTO_LIGHT + # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1209,7 +1344,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 270 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1227,15 +1362,6 @@ HTML_COLORSTYLE_SAT = 255 HTML_COLORSTYLE_GAMMA = 80 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via JavaScript. If disabled, the navigation index will @@ -1255,6 +1381,13 @@ HTML_DYNAMIC_MENUS = YES HTML_DYNAMIC_SECTIONS = NO +# If the HTML_CODE_FOLDING tag is set to YES then classes and functions can be +# dynamically folded and expanded in the generated HTML source code. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_CODE_FOLDING = YES + # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to @@ -1270,10 +1403,11 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/xcode/), introduced with OSX -# 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. @@ -1290,6 +1424,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1315,8 +1456,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1373,6 +1518,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -1391,7 +1546,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1399,8 +1555,8 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1408,16 +1564,16 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = @@ -1429,9 +1585,9 @@ QHP_CUST_FILTER_ATTRS = QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1474,16 +1630,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1508,6 +1676,13 @@ TREEVIEW_WIDTH = 250 EXT_LINKS_IN_WINDOW = NO +# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + # If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg # tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see # https://inkscape.org) to generate formulas as SVG images instead of PNGs for @@ -1528,17 +1703,6 @@ HTML_FORMULA_FORMAT = png FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - # The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands # to create new LaTeX commands to be used in formulas as building blocks. See # the section "Including formulas" for details. @@ -1556,11 +1720,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1573,22 +1755,29 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see +# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1635,7 +1824,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1648,8 +1838,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1758,29 +1949,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1823,10 +2016,16 @@ PDF_HYPERLINKS = YES USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# The LATEX_BATCHMODE tag signals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1839,16 +2038,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1857,14 +2046,6 @@ LATEX_SOURCE_CODE = NO LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_TIMESTAMP = NO - # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) # path from which the emoji images will be read. If a relative path is entered, # it will be relative to the LATEX_OUTPUT directory. If left blank the @@ -1929,16 +2110,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -1991,7 +2162,7 @@ MAN_LINKS = NO # captures the structure of the code including all documentation. # The default value is: NO. -GENERATE_XML = NO +GENERATE_XML = YES # The XML_OUTPUT tag is used to specify where the XML pages will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -2035,27 +2206,44 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures # the structure of the code including all documentation. Note that this feature # is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- + +# If the GENERATE_SQLITE3 tag is set to YES doxygen will generate a Sqlite3 +# database with symbols found by doxygen stored in tables. +# The default value is: NO. + +GENERATE_SQLITE3 = NO + +# The SQLITE3_OUTPUT tag is used to specify where the Sqlite3 database will be +# put. If a relative path is entered the value of OUTPUT_DIRECTORY will be put +# in front of it. +# The default directory is: sqlite3. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_OUTPUT = sqlite3 + +# The SQLITE3_OVERWRITE_DB tag is set to YES, the existing doxygen_sqlite3.db +# database file will be recreated with each doxygen run. If set to NO, doxygen +# will warn if an a database file is already found and not modify it. +# The default value is: YES. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_RECREATE_DB = YES + #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- @@ -2130,7 +2318,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -2197,15 +2386,15 @@ TAGFILES = rmm.tag=https://docs.rapids.ai/api/librmm/22.08 GENERATE_TAGFILE = -# If the ALLEXTERNALS tag is set to YES, all external class will be listed in -# the class index. If set to NO, only the inherited external classes will be -# listed. +# If the ALLEXTERNALS tag is set to YES, all external classes and namespaces +# will be listed in the class and namespace index. If set to NO, only the +# inherited external classes will be listed. # The default value is: NO. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will be +# in the topic index. If set to NO, only the current project's groups will be # listed. # The default value is: YES. @@ -2219,25 +2408,9 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. @@ -2246,7 +2419,7 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: NO. @@ -2263,49 +2436,73 @@ HAVE_DOT = YES DOT_NUM_THREADS = 0 -# When you want a differently looking font in the dot files that doxygen -# generates you can specify the font name using DOT_FONTNAME. You need to make -# sure dot is able to find the font, which can be done by putting it in a -# standard location or by setting the DOTFONTPATH environment variable or by -# setting DOT_FONTPATH to the directory containing the font. -# The default value is: Helvetica. +# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of +# subgraphs. When you want a differently looking font in the dot files that +# doxygen generates you can specify fontname, fontcolor and fontsize attributes. +# For details please see Node, +# Edge and Graph Attributes specification You need to make sure dot is able +# to find the font, which can be done by putting it in a standard location or by +# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. Default graphviz fontsize is 14. +# The default value is: fontname=Helvetica,fontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Helvetica +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" -# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of -# dot graphs. -# Minimum value: 4, maximum value: 24, default value: 10. +# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can +# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about +# arrows shapes. +# The default value is: labelfontname=Helvetica,labelfontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 10 +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" -# By default doxygen will tell dot to use the default font as specified with -# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set -# the path where dot can find it using this tag. +# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes +# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification +# The default value is: shape=box,height=0.2,width=0.4. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" + +# You can set the path where dot can find font specified with fontname in +# DOT_COMMON_ATTR and others dot attributes. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a # graph for each documented class showing the direct and indirect implementation # dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. +# class with other documented classes. Explicit enabling a collaboration graph, +# when COLLABORATION_GRAPH is set to NO, can be accomplished by means of the +# command \collaborationgraph. Disabling a collaboration graph can be +# accomplished by means of the command \hidecollaborationgraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# groups, showing the direct groups dependencies. Explicit enabling a group +# dependency graph, when GROUP_GRAPHS is set to NO, can be accomplished by means +# of the command \groupgraph. Disabling a directory graph can be accomplished by +# means of the command \hidegroupgraph. See also the chapter Grouping in the +# manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2328,10 +2525,32 @@ UML_LOOK = NO # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will wrapped across multiple lines. Some heuristics are apply +# to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2343,7 +2562,9 @@ TEMPLATE_RELATIONS = NO # If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to # YES then doxygen will generate a graph for each documented file showing the # direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an include graph, when INCLUDE_GRAPH is is set to NO, +# can be accomplished by means of the command \includegraph. Disabling an +# include graph can be accomplished by means of the command \hideincludegraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2352,7 +2573,10 @@ INCLUDE_GRAPH = YES # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are # set to YES then doxygen will generate a graph for each documented file showing # the direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an included by graph, when INCLUDED_BY_GRAPH is set +# to NO, can be accomplished by means of the command \includedbygraph. Disabling +# an included by graph can be accomplished by means of the command +# \hideincludedbygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2392,16 +2616,26 @@ GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the # dependencies a directory has on other directories in a graphical way. The # dependency relations are determined by the #include relations between the -# files in the directories. +# files in the directories. Explicit enabling a directory graph, when +# DIRECTORY_GRAPH is set to NO, can be accomplished by means of the command +# \directorygraph. Disabling a directory graph can be accomplished by means of +# the command \hidedirectorygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). +# https://www.graphviz.org/)). # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). @@ -2438,11 +2672,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2451,10 +2686,10 @@ MSCFILE_DIRS = DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = @@ -2492,18 +2727,6 @@ DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not seem -# to support this out of the box. -# -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_TRANSPARENT = NO - # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support @@ -2516,14 +2739,34 @@ DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/cpp/include/cugraph/algorithms.hpp b/cpp/include/cugraph/algorithms.hpp index 78846bc5766..8501eedce5c 100644 --- a/cpp/include/cugraph/algorithms.hpp +++ b/cpp/include/cugraph/algorithms.hpp @@ -464,51 +464,6 @@ k_truss_subgraph(raft::handle_t const& handle, size_t number_of_vertices, int k); -// FIXME: Internally distances is of int (signed 32-bit) data type, but current -// template uses data from VT, ET, WT from the legacy::GraphCSR View even if weights -// are not considered -/** - * @Synopsis Performs a breadth first search traversal of a graph starting from a vertex. - * - * @throws cugraph::logic_error with a custom message when an error occurs. - * - * @tparam VT Type of vertex identifiers. Supported value : int (signed, - * 32-bit) - * @tparam ET Type of edge identifiers. Supported value : int (signed, - * 32-bit) - * @tparam WT Type of edge weights. Supported values : int (signed, 32-bit) - * - * @param[in] handle Library handle (RAFT). If a communicator is set in the handle, - the multi GPU version will be selected. - * @param[in] graph cuGraph graph descriptor, should contain the connectivity - * information as a CSR - * - * @param[out] distances If set to a valid pointer, this is populated by distance of - * every vertex in the graph from the starting vertex - * - * @param[out] predecessors If set to a valid pointer, this is populated by bfs traversal - * predecessor of every vertex - * - * @param[out] sp_counters If set to a valid pointer, this is populated by bfs traversal - * shortest_path counter of every vertex - * - * @param[in] start_vertex The starting vertex for breadth first search traversal - * - * @param[in] directed Treat the input graph as directed - * - * @param[in] mg_batch If set to true use SG BFS path when comms are initialized. - * - */ -template -void bfs(raft::handle_t const& handle, - legacy::GraphCSRView const& graph, - VT* distances, - VT* predecessors, - double* sp_counters, - const VT start_vertex, - bool directed = true, - bool mg_batch = false); - /** * @brief Compute Hungarian algorithm on a weighted bipartite graph * diff --git a/cpp/include/cugraph/detail/collect_comm_wrapper.hpp b/cpp/include/cugraph/detail/collect_comm_wrapper.hpp new file mode 100644 index 00000000000..b791c593f41 --- /dev/null +++ b/cpp/include/cugraph/detail/collect_comm_wrapper.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include + +namespace cugraph { +namespace detail { + +/** + * @brief Gather the span of data from all ranks and broadcast the combined data to all ranks. + * + * @param[in] handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, + * and handles to various CUDA libraries) to run graph algorithms. + * @param[in] comm Raft comms that manages underlying NCCL comms handles across the ranks. + * @param[in] d_input The span of data to perform the 'allgatherv'. + * + * @return A vector containing the combined data of all ranks. + */ +template +rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +} // namespace detail +} // namespace cugraph diff --git a/cpp/include/cugraph/detail/decompress_edge_partition.cuh b/cpp/include/cugraph/detail/decompress_edge_partition.cuh index cd8739114f2..4b256a0413a 100644 --- a/cpp/include/cugraph/detail/decompress_edge_partition.cuh +++ b/cpp/include/cugraph/detail/decompress_edge_partition.cuh @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -46,7 +47,7 @@ __global__ void decompress_to_edgelist_mid_degree( edge_partition_device_view_t edge_partition, vertex_t major_range_first, vertex_t major_range_last, - vertex_t* majors) + raft::device_span majors) { auto const tid = threadIdx.x + blockIdx.x * blockDim.x; static_assert(decompress_edge_partition_block_size % raft::warp_size() == 0); @@ -76,7 +77,7 @@ __global__ void decompress_to_edgelist_high_degree( edge_partition_device_view_t edge_partition, vertex_t major_range_first, vertex_t major_range_last, - vertex_t* majors) + raft::device_span majors) { auto major_start_offset = static_cast(major_range_first - edge_partition.major_range_first()); @@ -103,10 +104,19 @@ template void decompress_edge_partition_to_fill_edgelist_majors( raft::handle_t const& handle, edge_partition_device_view_t edge_partition, - vertex_t* majors, + std::optional> + edge_partition_mask_view, + raft::device_span majors, std::optional> const& segment_offsets) { - auto execution_policy = handle.get_thrust_policy(); + auto tmp_buffer = edge_partition_mask_view + ? std::make_optional>( + edge_partition.number_of_edges(), handle.get_stream()) + : std::nullopt; + + auto output_buffer = + tmp_buffer ? raft::device_span((*tmp_buffer).data(), (*tmp_buffer).size()) : majors; + if (segment_offsets) { // FIXME: we may further improve performance by 1) concurrently running kernels on different // segments; 2) individually tuning block sizes for different segments; and 3) adding one more @@ -124,7 +134,7 @@ void decompress_edge_partition_to_fill_edgelist_majors( edge_partition, edge_partition.major_range_first(), edge_partition.major_range_first() + (*segment_offsets)[1], - majors); + output_buffer); } if ((*segment_offsets)[2] - (*segment_offsets)[1] > 0) { raft::grid_1d_warp_t update_grid((*segment_offsets)[2] - (*segment_offsets)[1], @@ -138,49 +148,63 @@ void decompress_edge_partition_to_fill_edgelist_majors( edge_partition, edge_partition.major_range_first() + (*segment_offsets)[1], edge_partition.major_range_first() + (*segment_offsets)[2], - majors); + output_buffer); } if ((*segment_offsets)[3] - (*segment_offsets)[2] > 0) { thrust::for_each( - execution_policy, + handle.get_thrust_policy(), thrust::make_counting_iterator(edge_partition.major_range_first()) + (*segment_offsets)[2], thrust::make_counting_iterator(edge_partition.major_range_first()) + (*segment_offsets)[3], - [edge_partition, majors] __device__(auto major) { + [edge_partition, output_buffer] __device__(auto major) { auto major_offset = edge_partition.major_offset_from_major_nocheck(major); auto local_degree = edge_partition.local_degree(major_offset); auto local_offset = edge_partition.local_offset(major_offset); - thrust::fill( - thrust::seq, majors + local_offset, majors + local_offset + local_degree, major); + thrust::fill(thrust::seq, + output_buffer.begin() + local_offset, + output_buffer.begin() + local_offset + local_degree, + major); }); } if (edge_partition.dcs_nzd_vertex_count() && (*(edge_partition.dcs_nzd_vertex_count()) > 0)) { thrust::for_each( - execution_policy, + handle.get_thrust_policy(), thrust::make_counting_iterator(vertex_t{0}), thrust::make_counting_iterator(*(edge_partition.dcs_nzd_vertex_count())), - [edge_partition, major_start_offset = (*segment_offsets)[3], majors] __device__(auto idx) { + [edge_partition, major_start_offset = (*segment_offsets)[3], output_buffer] __device__( + auto idx) { auto major = *(edge_partition.major_from_major_hypersparse_idx_nocheck(idx)); auto major_idx = major_start_offset + idx; // major_offset != major_idx in the hypersparse region auto local_degree = edge_partition.local_degree(major_idx); auto local_offset = edge_partition.local_offset(major_idx); - thrust::fill( - thrust::seq, majors + local_offset, majors + local_offset + local_degree, major); + thrust::fill(thrust::seq, + output_buffer.begin() + local_offset, + output_buffer.begin() + local_offset + local_degree, + major); }); } } else { - thrust::for_each( - execution_policy, - thrust::make_counting_iterator(edge_partition.major_range_first()), - thrust::make_counting_iterator(edge_partition.major_range_first()) + - edge_partition.major_range_size(), - [edge_partition, majors] __device__(auto major) { - auto major_offset = edge_partition.major_offset_from_major_nocheck(major); - auto local_degree = edge_partition.local_degree(major_offset); - auto local_offset = edge_partition.local_offset(major_offset); - thrust::fill( - thrust::seq, majors + local_offset, majors + local_offset + local_degree, major); - }); + thrust::for_each(handle.get_thrust_policy(), + thrust::make_counting_iterator(edge_partition.major_range_first()), + thrust::make_counting_iterator(edge_partition.major_range_first()) + + edge_partition.major_range_size(), + [edge_partition, output_buffer] __device__(auto major) { + auto major_offset = edge_partition.major_offset_from_major_nocheck(major); + auto local_degree = edge_partition.local_degree(major_offset); + auto local_offset = edge_partition.local_offset(major_offset); + thrust::fill(thrust::seq, + output_buffer.begin() + local_offset, + output_buffer.begin() + local_offset + local_degree, + major); + }); + } + + if (tmp_buffer) { + copy_if_mask_set(handle, + (*tmp_buffer).begin(), + (*tmp_buffer).end(), + (*edge_partition_mask_view).value_first(), + majors.begin()); } } @@ -192,33 +216,59 @@ void decompress_edge_partition_to_edgelist( edge_partition_weight_view, std::optional> edge_partition_id_view, - vertex_t* edgelist_majors /* [OUT] */, - vertex_t* edgelist_minors /* [OUT] */, - std::optional edgelist_weights /* [OUT] */, - std::optional edgelist_ids /* [OUT] */, + std::optional> + edge_partition_mask_view, + raft::device_span edgelist_majors /* [OUT] */, + raft::device_span edgelist_minors /* [OUT] */, + std::optional> edgelist_weights /* [OUT] */, + std::optional> edgelist_ids /* [OUT] */, std::optional> const& segment_offsets) { auto number_of_edges = edge_partition.number_of_edges(); decompress_edge_partition_to_fill_edgelist_majors( - handle, edge_partition, edgelist_majors, segment_offsets); - thrust::copy(handle.get_thrust_policy(), - edge_partition.indices(), - edge_partition.indices() + number_of_edges, - edgelist_minors); - if (edge_partition_id_view) { - assert(edgelist_ids.has_value()); + handle, edge_partition, edge_partition_mask_view, edgelist_majors, segment_offsets); + if (edge_partition_mask_view) { + copy_if_mask_set(handle, + edge_partition.indices(), + edge_partition.indices() + number_of_edges, + (*edge_partition_mask_view).value_first(), + edgelist_minors.begin()); + } else { thrust::copy(handle.get_thrust_policy(), - (*edge_partition_id_view).value_first(), - (*edge_partition_id_view).value_first() + number_of_edges, - (*edgelist_ids)); + edge_partition.indices(), + edge_partition.indices() + number_of_edges, + edgelist_minors.begin()); } if (edge_partition_weight_view) { assert(edgelist_weights.has_value()); - thrust::copy(handle.get_thrust_policy(), - (*edge_partition_weight_view).value_first(), - (*edge_partition_weight_view).value_first() + number_of_edges, - (*edgelist_weights)); + if (edge_partition_mask_view) { + copy_if_mask_set(handle, + (*edge_partition_weight_view).value_first(), + (*edge_partition_weight_view).value_first() + number_of_edges, + (*edge_partition_mask_view).value_first(), + (*edgelist_weights).begin()); + } else { + thrust::copy(handle.get_thrust_policy(), + (*edge_partition_weight_view).value_first(), + (*edge_partition_weight_view).value_first() + number_of_edges, + (*edgelist_weights).begin()); + } + } + if (edge_partition_id_view) { + assert(edgelist_ids.has_value()); + if (edge_partition_mask_view) { + copy_if_mask_set(handle, + (*edge_partition_id_view).value_first(), + (*edge_partition_id_view).value_first() + number_of_edges, + (*edge_partition_mask_view).value_first(), + (*edgelist_ids).begin()); + } else { + thrust::copy(handle.get_thrust_policy(), + (*edge_partition_id_view).value_first(), + (*edge_partition_id_view).value_first() + number_of_edges, + (*edgelist_ids).begin()); + } } } diff --git a/cpp/include/cugraph/edge_partition_device_view.cuh b/cpp/include/cugraph/edge_partition_device_view.cuh index d34d639f4d9..213f9b9497a 100644 --- a/cpp/include/cugraph/edge_partition_device_view.cuh +++ b/cpp/include/cugraph/edge_partition_device_view.cuh @@ -109,6 +109,13 @@ class edge_partition_device_view_base_t { __host__ __device__ edge_t const* offsets() const { return offsets_.data(); } __host__ __device__ vertex_t const* indices() const { return indices_.data(); } + __device__ vertex_t major_idx_from_local_edge_idx_nocheck(edge_t local_edge_idx) const noexcept + { + return static_cast(thrust::distance( + offsets_.begin() + 1, + thrust::upper_bound(thrust::seq, offsets_.begin() + 1, offsets_.end(), local_edge_idx))); + } + // major_idx == major offset if CSR/CSC, major_offset != major_idx if DCSR/DCSC __device__ thrust::tuple local_edges( vertex_t major_idx) const noexcept @@ -291,8 +298,19 @@ class edge_partition_device_view_t= (*major_hypersparse_first_ - major_range_first_) + ? (*dcs_nzd_vertices_)[major_idx - (*major_hypersparse_first_ - major_range_first_)] + : major_from_major_offset_nocheck(major_idx); + } else { // major_idx == major_offset + return major_from_major_offset_nocheck(major_idx); + } + } + // major_hypersparse_idx: index within the hypersparse segment - __host__ __device__ thrust::optional major_hypersparse_idx_from_major_nocheck( + __device__ thrust::optional major_hypersparse_idx_from_major_nocheck( vertex_t major) const noexcept { if (dcs_nzd_vertices_) { @@ -303,7 +321,7 @@ class edge_partition_device_view_t major_from_major_hypersparse_idx_nocheck( + __device__ thrust::optional major_from_major_hypersparse_idx_nocheck( vertex_t major_hypersparse_idx) const noexcept { return dcs_nzd_vertices_ @@ -442,8 +460,13 @@ class edge_partition_device_view_t major_hypersparse_idx_from_major_nocheck( + __device__ thrust::optional major_hypersparse_idx_from_major_nocheck( vertex_t major) const noexcept { assert(false); @@ -451,7 +474,7 @@ class edge_partition_device_view_t major_from_major_hypersparse_idx_nocheck( + __device__ thrust::optional major_from_major_hypersparse_idx_nocheck( vertex_t major_hypersparse_idx) const noexcept { assert(false); diff --git a/cpp/include/cugraph/edge_partition_edge_property_device_view.cuh b/cpp/include/cugraph/edge_partition_edge_property_device_view.cuh index f71fc167d12..e5b64b1e02f 100644 --- a/cpp/include/cugraph/edge_partition_edge_property_device_view.cuh +++ b/cpp/include/cugraph/edge_partition_edge_property_device_view.cuh @@ -33,29 +33,33 @@ template ::value_type> class edge_partition_edge_property_device_view_t { public: + using edge_type = edge_t; + using value_type = value_t; + + static constexpr bool is_packed_bool = cugraph::is_packed_bool(); + static constexpr bool has_packed_bool_element = + cugraph::has_packed_bool_element(); + static_assert( std::is_same_v::value_type, value_t> || - cugraph::has_packed_bool_element()); + has_packed_bool_element); static_assert(cugraph::is_arithmetic_or_thrust_tuple_of_arithmetic::value); - using edge_type = edge_t; - using value_type = value_t; - edge_partition_edge_property_device_view_t() = default; edge_partition_edge_property_device_view_t( - edge_property_view_t const& view, size_t partition_idx) + edge_property_view_t const& view, size_t partition_idx) : value_first_(view.value_firsts()[partition_idx]) { value_first_ = view.value_firsts()[partition_idx]; } - __host__ __device__ ValueIterator value_first() { return value_first_; } + __host__ __device__ ValueIterator value_first() const { return value_first_; } __device__ value_t get(edge_t offset) const { - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(offset); return static_cast(*(value_first_ + cugraph::packed_bool_offset(offset)) & mask); } else { @@ -69,8 +73,8 @@ class edge_partition_edge_property_device_view_t { void> set(edge_t offset, value_t val) const { - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(offset); if (val) { atomicOr(value_first_ + cugraph::packed_bool_offset(offset), mask); @@ -88,8 +92,8 @@ class edge_partition_edge_property_device_view_t { value_t> atomic_and(edge_t offset, value_t val) const { - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(offset); auto old = atomicAnd(value_first_ + cugraph::packed_bool_offset(offset), val ? uint32_t{0xffffffff} : ~mask); @@ -105,8 +109,8 @@ class edge_partition_edge_property_device_view_t { value_t> atomic_or(edge_t offset, value_t val) const { - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(offset); auto old = atomicOr(value_first_ + cugraph::packed_bool_offset(offset), val ? mask : uint32_t{0}); @@ -132,8 +136,8 @@ class edge_partition_edge_property_device_view_t { value_t> elementwise_atomic_cas(edge_t offset, value_t compare, value_t val) const { - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(offset); auto old = val ? atomicOr(value_first_ + cugraph::packed_bool_offset(offset), mask) : atomicAnd(value_first_ + cugraph::packed_bool_offset(offset), ~mask); @@ -173,6 +177,9 @@ class edge_partition_edge_dummy_property_device_view_t { using edge_type = edge_t; using value_type = thrust::nullopt_t; + static constexpr bool is_packed_bool = false; + static constexpr bool has_packed_bool_element = false; + edge_partition_edge_dummy_property_device_view_t() = default; edge_partition_edge_dummy_property_device_view_t(edge_dummy_property_view_t const& view, diff --git a/cpp/include/cugraph/edge_partition_endpoint_property_device_view.cuh b/cpp/include/cugraph/edge_partition_endpoint_property_device_view.cuh index 1ff279fbdca..7578c646175 100644 --- a/cpp/include/cugraph/edge_partition_endpoint_property_device_view.cuh +++ b/cpp/include/cugraph/edge_partition_endpoint_property_device_view.cuh @@ -39,12 +39,15 @@ template ::value_type> class edge_partition_endpoint_property_device_view_t { public: + using vertex_type = vertex_t; + using value_type = value_t; + static constexpr bool is_packed_bool = cugraph::is_packed_bool(); + static constexpr bool has_packed_bool_element = + cugraph::has_packed_bool_element(); + static_assert( std::is_same_v::value_type, value_t> || - cugraph::has_packed_bool_element()); - - using vertex_type = vertex_t; - using value_type = value_t; + has_packed_bool_element); edge_partition_endpoint_property_device_view_t() = default; @@ -77,8 +80,8 @@ class edge_partition_endpoint_property_device_view_t { __device__ value_t get(vertex_t offset) const { auto val_offset = value_offset(offset); - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(val_offset); return static_cast(*(value_first_ + cugraph::packed_bool_offset(val_offset)) & mask); } else { @@ -93,8 +96,8 @@ class edge_partition_endpoint_property_device_view_t { atomic_and(vertex_t offset, value_t val) const { auto val_offset = value_offset(offset); - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(val_offset); auto old = atomicAnd(value_first_ + cugraph::packed_bool_offset(val_offset), val ? cugraph::packed_bool_full_mask() : ~mask); @@ -111,8 +114,8 @@ class edge_partition_endpoint_property_device_view_t { atomic_or(vertex_t offset, value_t val) const { auto val_offset = value_offset(offset); - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(val_offset); auto old = atomicOr(value_first_ + cugraph::packed_bool_offset(val_offset), val ? mask : cugraph::packed_bool_empty_mask()); @@ -140,8 +143,8 @@ class edge_partition_endpoint_property_device_view_t { elementwise_atomic_cas(vertex_t offset, value_t compare, value_t val) const { auto val_offset = value_offset(offset); - if constexpr (cugraph::has_packed_bool_element()) { - static_assert(std::is_arithmetic_v, "unimplemented for thrust::tuple types."); + if constexpr (has_packed_bool_element) { + static_assert(is_packed_bool, "unimplemented for thrust::tuple types."); auto mask = cugraph::packed_bool_mask(val_offset); auto old = val ? atomicOr(value_first_ + cugraph::packed_bool_offset(val_offset), mask) : atomicAnd(value_first_ + cugraph::packed_bool_offset(val_offset), ~mask); @@ -203,8 +206,10 @@ class edge_partition_endpoint_property_device_view_t { template class edge_partition_endpoint_dummy_property_device_view_t { public: - using vertex_type = vertex_t; - using value_type = thrust::nullopt_t; + using vertex_type = vertex_t; + using value_type = thrust::nullopt_t; + static constexpr bool is_packed_bool = false; + static constexpr bool has_packed_bool_element = false; edge_partition_endpoint_dummy_property_device_view_t() = default; diff --git a/cpp/include/cugraph/edge_property.hpp b/cpp/include/cugraph/edge_property.hpp index 8904006a2a2..d46d4e52fd4 100644 --- a/cpp/include/cugraph/edge_property.hpp +++ b/cpp/include/cugraph/edge_property.hpp @@ -72,9 +72,11 @@ class edge_property_t { public: static_assert(cugraph::is_arithmetic_or_thrust_tuple_of_arithmetic::value); - using edge_type = typename GraphViewType::edge_type; - using value_type = T; - using buffer_type = decltype(allocate_dataframe_buffer(size_t{0}, rmm::cuda_stream_view{})); + using edge_type = typename GraphViewType::edge_type; + using value_type = T; + using buffer_type = + decltype(allocate_dataframe_buffer, uint32_t, T>>( + size_t{0}, rmm::cuda_stream_view{})); edge_property_t(raft::handle_t const& handle) {} diff --git a/cpp/include/cugraph/graph_functions.hpp b/cpp/include/cugraph/graph_functions.hpp index 5c1e9d5311f..6a75a420bf8 100644 --- a/cpp/include/cugraph/graph_functions.hpp +++ b/cpp/include/cugraph/graph_functions.hpp @@ -973,4 +973,71 @@ renumber_sampled_edgelist( label_offsets, bool do_expensive_check = false); +/** + * @brief Remove self loops from an edge list + * + * @tparam vertex_t Type of vertex identifiers. Needs to be an integral type. + * @tparam edge_t Type of edge identifiers. Needs to be an integral type. + * @tparam weight_t Type of edge weight. Currently float and double are supported. + * @tparam edge_type_t Type of edge type. Needs to be an integral type. + * + * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and + * handles to various CUDA libraries) to run graph algorithms. + * @param edgelist_srcs List of source vertex ids + * @param edgelist_dsts List of destination vertex ids + * @param edgelist_weights Optional list of edge weights + * @param edgelist_edge_ids Optional list of edge ids + * @param edgelist_edge_types Optional list of edge types + * @return Tuple of vectors storing edge sources, destinations, optional weights, + * optional edge ids, optional edge types. + */ +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +/** + * @brief Remove all but one edge when a multi-edge exists. Note that this function does not use + * stable methods. When a multi-edge exists, one of the edges will remain, there is no + * guarantee on which one will remain. + * + * In an MG context it is assumed that edges have been shuffled to the proper GPU, + * in which case any multi-edges will be on the same GPU. + * + * @tparam vertex_t Type of vertex identifiers. Needs to be an integral type. + * @tparam edge_t Type of edge identifiers. Needs to be an integral type. + * @tparam weight_t Type of edge weight. Currently float and double are supported. + * @tparam edge_type_t Type of edge type. Needs to be an integral type. + * + * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and + * handles to various CUDA libraries) to run graph algorithms. + * @param edgelist_srcs List of source vertex ids + * @param edgelist_dsts List of destination vertex ids + * @param edgelist_weights Optional list of edge weights + * @param edgelist_edge_ids Optional list of edge ids + * @param edgelist_edge_types Optional list of edge types + * @return Tuple of vectors storing edge sources, destinations, optional weights, + * optional edge ids, optional edge types. + */ +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + } // namespace cugraph diff --git a/cpp/include/cugraph/graph_mask.hpp b/cpp/include/cugraph/graph_mask.hpp deleted file mode 100644 index 2048d3692c7..00000000000 --- a/cpp/include/cugraph/graph_mask.hpp +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include - -namespace cugraph { - -/** - * Compile-time fast lookup of log2(num_bits(mask_t)) to eliminate - * the log2 computation for powers of 2. - * @tparam mask_t - */ -template -__host__ __device__ constexpr int log_bits() -{ - switch (std::numeric_limits::digits) { - case 8: return 3; - case 16: return 4; - case 32: return 5; - case 64: return 6; - default: return log2(std::numeric_limits::digits); - } -} - -/** - * Uses bit-shifting to perform a fast mod operation. This - * is used to compute the index of a specific bit - * @tparam mask_t - * @tparam T - */ -template -__host__ __device__ int bit_mod(T numerator) -{ - return numerator & (std::numeric_limits::digits - 1); -} - -namespace detail { - -/** - * Sets the bit at location h in a one-hot encoded 32-bit int array - */ -template -__device__ __host__ inline void _set_bit(mask_type* arr, mask_type h) -{ - mask_type bit = bit_mod(h); - mask_type idx = h >> log_bits(); - atomicOr(arr + idx, 1 << bit); -} - -/** - * Unsets the bit at location h in a one-hot encoded 32-bit int array - */ -template -__device__ __host__ inline void _unset_bit(mask_type* arr, mask_type h) -{ - mask_type bit = bit_mod(h); - mask_type idx = h >> log_bits(); - atomicAnd(arr + idx, ~(1 << bit)); -} - -/** - * Returns whether or not bit at location h is nonzero in a one-hot - * encoded 32-bit in array. - */ -template -__device__ __host__ inline bool _is_set(mask_type* arr, mask_type h) -{ - mask_type bit = bit_mod(h); - mask_type idx = h >> log_bits(); - return arr[idx] >> bit & 1U; -} -}; // namespace detail - -/** - * Mask view to be used in device functions for reading and updating existing mask. - * This assumes the appropriate masks (vertex/edge) have already been initialized, - * since that will need to be done from the owning object. - * @tparam vertex_t - * @tparam edge_t - * @tparam mask_t - */ -template -struct graph_mask_view_t { - public: - graph_mask_view_t() = delete; - - graph_mask_view_t(vertex_t n_vertices, - edge_t n_edges, - std::optional>& vertices, - std::optional>& edges, - bool complement = false) - : n_vertices_(n_vertices), - n_edges_(n_edges), - complement_(complement), - vertices_(vertices), - edges_(edges) - { - } - - graph_mask_view_t(graph_mask_view_t const& other) = default; - using vertex_type = vertex_t; - using edge_type = edge_t; - using mask_type = mask_t; - using size_type = std::size_t; - - ~graph_mask_view_t() = default; - graph_mask_view_t(graph_mask_view_t&&) noexcept = default; - - graph_mask_view_t& operator=(graph_mask_view_t&&) noexcept = default; - graph_mask_view_t& operator=(graph_mask_view_t const& other) = default; - - /** - * Are masks complemeneted? - * - * - !complemented means masks are inclusive (masking in) - * - complemented means masks are exclusive (masking out) - */ - __host__ __device__ bool is_complemented() const { return complement_; } - - /** - * Has the edge mask been initialized? - */ - __host__ __device__ bool has_edge_mask() const { return edges_.has_value(); } - - /** - * Has the vertex mask been initialized? - */ - __host__ __device__ bool has_vertex_mask() const { return vertices_.has_value(); } - - /** - * Get the vertex mask - */ - __host__ __device__ std::optional> get_vertex_mask() const - { - return vertices_; - } - - /** - * Get the edge mask - */ - __host__ __device__ std::optional> get_edge_mask() const - { - return edges_; - } - - __host__ __device__ edge_t get_edge_mask_size() const { return n_edges_ >> log_bits(); } - - __host__ __device__ vertex_t get_vertex_mask_size() const - { - return n_vertices_ >> log_bits(); - } - - protected: - vertex_t n_vertices_; - edge_t n_edges_; - bool complement_{false}; - std::optional> vertices_{std::nullopt}; - std::optional> edges_{std::nullopt}; -}; // struct graph_mask_view_t - -/** - * An owning container object to manage separate bitmasks for - * filtering vertices and edges. A compliment setting - * determines whether the value of 1 for corresponding - * items means they should be masked in (included) or - * masked out (excluded). - * - * Creating this object does not allocate any memory on device. - * In order to start using and querying the masks, they will - * need to first be initialized. - * - * @tparam vertex_t - * @tparam edge_t - * @tparam mask_t - */ -template -struct graph_mask_t { - public: - using vertex_type = vertex_t; - using edge_type = edge_t; - using mask_type = mask_t; - using size_type = std::size_t; - - ~graph_mask_t() = default; - graph_mask_t(graph_mask_t&&) noexcept = default; - - graph_mask_t() = delete; - - explicit graph_mask_t(raft::handle_t const& handle, - vertex_t n_vertices, - edge_t n_edges, - bool complement = false) - : handle_(handle), - n_vertices_(n_vertices), - n_edges_(n_edges), - edges_(0, handle.get_stream()), - vertices_(0, handle.get_stream()), - complement_(complement) - { - } - - explicit graph_mask_t(graph_mask_t const& other) - : handle_(other.handle_), - n_vertices_(other.n_vertices_), - n_edges_(other.n_edges_), - edges_(other.edges_, other.handle_.get_stream()), - vertices_(other.vertices_, other.handle_.get_stream()), - complement_(other.complement_) - { - } - - graph_mask_t& operator=(graph_mask_t&&) noexcept = default; - graph_mask_t& operator=(graph_mask_t const& other) = default; - - /** - * Determines whether the 1 bit in a vertex or edge position - * represents an inclusive mask or exclusive mask. Default is - * an inclusive mask (e.g. 1 bit means the corresponding vertex - * or edge should be included in computations). - * @return - */ - bool is_complemented() const { return complement_; } - - /** - * Whether or not the current mask object has been initialized - * with an edge mask. - * @return - */ - bool has_edge_mask() const { return get_edge_mask().has_value(); } - - /** - * Whether or not the current mask object has been initialized - * with a vertex mask. - * @return - */ - bool has_vertex_mask() const { return get_vertex_mask().has_value(); } - - /** - * Returns the edge mask if it has been initialized on the instance - * @return - */ - std::optional get_edge_mask() const - { - return edges_.size() > 0 ? std::make_optional(edges_.data()) : std::nullopt; - } - - /** - * Retuns the vertex mask if it has been initialized on the instance - * @return - */ - std::optional get_vertex_mask() const - { - return vertices_.size() > 0 ? std::make_optional(vertices_.data()) - : std::nullopt; - } - - vertex_t get_n_vertices() { return n_vertices_; } - - edge_t get_n_edges() { return n_edges_; } - - edge_t get_edge_mask_size() const { return n_edges_ >> log_bits(); } - - vertex_t get_vertex_mask_size() const { return n_vertices_ >> log_bits(); } - - void initialize_edge_mask(bool init = 0) - { - if (!has_edge_mask()) { - allocate_edge_mask(); - RAFT_CUDA_TRY(cudaMemsetAsync(edges_.data(), - edges_.size() * sizeof(mask_t), - std::numeric_limits::max() * init, - handle_.get_stream())); - } - } - - void initialize_vertex_mask(bool init = 0) - { - if (!has_vertex_mask()) { - allocate_vertex_mask(); - RAFT_CUDA_TRY(cudaMemsetAsync(vertices_.data(), - vertices_.size() * sizeof(mask_t), - std::numeric_limits::max() * init, - handle_.get_stream())); - } - } - - /** - * Initializes an edge mask by allocating the device memory - */ - void allocate_edge_mask() - { - if (edges_.size() == 0) { - edges_.resize(get_edge_mask_size(), handle_.get_stream()); - clear_edge_mask(); - } - } - - /** - * Initializes a vertex mask by allocating the device memory - */ - void allocate_vertex_mask() - { - if (vertices_.size() == 0) { - vertices_.resize(get_vertex_mask_size(), handle_.get_stream()); - clear_vertex_mask(); - } - } - - /** - * Clears out all the masked bits of the edge mask - */ - void clear_edge_mask() - { - if (edges_.size() > 0) { - RAFT_CUDA_TRY( - cudaMemsetAsync(edges_.data(), edges_.size() * sizeof(mask_t), 0, handle_.get_stream())); - } - } - - /** - * Clears out all the masked bits of the vertex mask - */ - void clear_vertex_mask() - { - if (vertices_.size() > 0) { - RAFT_CUDA_TRY(cudaMemsetAsync( - vertices_.data(), vertices_.size() * sizeof(mask_t), 0, handle_.get_stream())); - } - } - - /** - * Returns a view of the current mask object which can safely be used - * in device functions. - * - * Note that the view will not be able to initialize the underlying - * masks so they will need to be initialized before this method is - * invoked. - */ - auto view() - { - auto vspan = has_vertex_mask() ? std::make_optional>(vertices_.data(), - vertices_.size()) - : std::nullopt; - auto espan = has_edge_mask() - ? std::make_optional>(edges_.data(), edges_.size()) - : std::nullopt; - return graph_mask_view_t( - n_vertices_, n_edges_, vspan, espan, complement_); - } - - protected: - raft::handle_t const& handle_; - vertex_t n_vertices_; - edge_t n_edges_; - bool complement_ = false; - rmm::device_uvector vertices_; - rmm::device_uvector edges_; - -}; // struct graph_mask_t -}; // namespace cugraph \ No newline at end of file diff --git a/cpp/include/cugraph/graph_view.hpp b/cpp/include/cugraph/graph_view.hpp index 2d10b435224..f30a8b7e2af 100644 --- a/cpp/include/cugraph/graph_view.hpp +++ b/cpp/include/cugraph/graph_view.hpp @@ -444,12 +444,6 @@ class graph_view_t local_edge_partition_view( size_t partition_idx) const { - CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); - vertex_t major_range_first{}; vertex_t major_range_last{}; vertex_t minor_range_first{}; @@ -748,6 +740,11 @@ class graph_view_t> edge_mask_view() const + { + return edge_mask_view_; + } + private: std::vector edge_partition_offsets_{}; std::vector edge_partition_indices_{}; @@ -856,12 +853,6 @@ class graph_view_tnumber_of_edges(); - } - vertex_t local_edge_partition_src_range_size(size_t partition_idx = 0) const { assert(partition_idx == 0); @@ -1030,6 +1021,11 @@ class graph_view_t> edge_mask_view() const + { + return edge_mask_view_; + } + private: edge_t const* offsets_{nullptr}; vertex_t const* indices_{nullptr}; diff --git a/cpp/include/cugraph/mtmg/detail/per_device_edgelist.hpp b/cpp/include/cugraph/mtmg/detail/per_device_edgelist.hpp index 8011146ee4f..7fd5bb726e6 100644 --- a/cpp/include/cugraph/mtmg/detail/per_device_edgelist.hpp +++ b/cpp/include/cugraph/mtmg/detail/per_device_edgelist.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include // FIXME: Could use std::span once compiler supports C++20 @@ -58,6 +59,15 @@ class per_device_edgelist_t { per_device_edgelist_t& operator=(per_device_edgelist_t const&) = delete; per_device_edgelist_t& operator=(per_device_edgelist_t&&) = delete; + /** + * @brief Construct a new per device edgelist t object + * + * @param handle MTMG resource handle - used to identify GPU resources + * @param device_buffer_size Number of edges to store in each device buffer + * @param use_weight Whether or not the edgelist will have weights + * @param use_edge_id Whether or not the edgelist will have edge ids + * @param use_edge_type Whether or not the edgelist will have edge types + */ per_device_edgelist_t(cugraph::mtmg::handle_t const& handle, size_t device_buffer_size, bool use_weight, @@ -82,6 +92,11 @@ class per_device_edgelist_t { create_new_buffers(handle); } + /** + * @brief Move construct a new per device edgelist t object + * + * @param other Object to move into this instance + */ per_device_edgelist_t(per_device_edgelist_t&& other) : device_buffer_size_{other.device_buffer_size_}, current_pos_{other.current_pos_}, @@ -110,45 +125,72 @@ class per_device_edgelist_t { std::optional> edge_id, std::optional> edge_type) { - // FIXME: This lock guard could be on a smaller region, but it - // would require more careful coding. The raft::update_device - // calls could be done without the lock if we made a local - // of the values of *.back() and did an increment of current_pos_ - // while we hold the lock. - std::lock_guard lock(lock_); - - size_t count = src.size(); - size_t pos = 0; - - while (count > 0) { - size_t copy_count = std::min(count, (src_.back().size() - current_pos_)); - - raft::update_device( - src_.back().begin() + current_pos_, src.begin() + pos, copy_count, handle.get_stream()); - raft::update_device( - dst_.back().begin() + current_pos_, dst.begin() + pos, copy_count, handle.get_stream()); - if (wgt) - raft::update_device( - wgt_->back().begin() + current_pos_, wgt->begin() + pos, copy_count, handle.get_stream()); - if (edge_id) - raft::update_device(edge_id_->back().begin() + current_pos_, - edge_id->begin() + pos, - copy_count, - handle.get_stream()); - if (edge_type) - raft::update_device(edge_type_->back().begin() + current_pos_, - edge_type->begin() + pos, - copy_count, - handle.get_stream()); - - count -= copy_count; - pos += copy_count; - current_pos_ += copy_count; - - if (current_pos_ == src_.back().size()) { create_new_buffers(handle); } + std::vector> copy_positions; + + { + std::lock_guard lock(lock_); + + size_t count = src.size(); + size_t pos = 0; + + while (count > 0) { + size_t copy_count = std::min(count, (src_.back().size() - current_pos_)); + + copy_positions.push_back(std::make_tuple(src_.size() - 1, current_pos_, pos, copy_count)); + + count -= copy_count; + pos += copy_count; + current_pos_ += copy_count; + + if (current_pos_ == src_.back().size()) { create_new_buffers(handle); } + } } - handle.raft_handle().sync_stream(); + std::for_each(copy_positions.begin(), + copy_positions.end(), + [&handle, + &this_src = src_, + &src, + &this_dst = dst_, + &dst, + &this_wgt = wgt_, + &wgt, + &this_edge_id = edge_id_, + &edge_id, + &this_edge_type = edge_type_, + &edge_type](auto tuple) { + auto [buffer_idx, buffer_pos, input_pos, copy_count] = tuple; + + raft::update_device(this_src[buffer_idx].begin() + buffer_pos, + src.begin() + input_pos, + copy_count, + handle.get_stream()); + + raft::update_device(this_dst[buffer_idx].begin() + buffer_pos, + dst.begin() + input_pos, + copy_count, + handle.get_stream()); + + if (this_wgt) + raft::update_device((*this_wgt)[buffer_idx].begin() + buffer_pos, + wgt->begin() + input_pos, + copy_count, + handle.get_stream()); + + if (this_edge_id) + raft::update_device((*this_edge_id)[buffer_idx].begin() + buffer_pos, + edge_id->begin() + input_pos, + copy_count, + handle.get_stream()); + + if (this_edge_type) + raft::update_device((*this_edge_type)[buffer_idx].begin() + buffer_pos, + edge_type->begin() + input_pos, + copy_count, + handle.get_stream()); + }); + + handle.sync_stream(); } /** diff --git a/cpp/include/cugraph/mtmg/handle.hpp b/cpp/include/cugraph/mtmg/handle.hpp index f23bce5aeac..6223de1781d 100644 --- a/cpp/include/cugraph/mtmg/handle.hpp +++ b/cpp/include/cugraph/mtmg/handle.hpp @@ -18,6 +18,8 @@ #include +#include + namespace cugraph { namespace mtmg { @@ -60,10 +62,41 @@ class handle_t { rmm::cuda_stream_view get_stream() const { return raft_handle_.is_stream_pool_initialized() - ? raft_handle_.get_stream_from_stream_pool(device_id_) + ? raft_handle_.get_stream_from_stream_pool(thread_rank_) : raft_handle_.get_stream(); } + /** + * @brief Sync on the cuda stream + * + * @param stream Which stream to synchronize (defaults to the stream for this handle) + */ + void sync_stream(rmm::cuda_stream_view stream) const { raft_handle_.sync_stream(stream); } + + /** + * @brief Sync on the cuda stream for this handle + */ + void sync_stream() const { sync_stream(get_stream()); } + + /** + * @brief get thrust policy for the stream + * + * @param stream Which stream to use for this thrust call + * + * @return exec policy using the current stream + */ + rmm::exec_policy get_thrust_policy(rmm::cuda_stream_view stream) const + { + return rmm::exec_policy(stream); + } + + /** + * @brief get thrust policy for the stream for this handle + * + * @return exec policy using the current stream + */ + rmm::exec_policy get_thrust_policy() const { return get_thrust_policy(get_stream()); } + /** * @brief Get thread rank * @@ -78,14 +111,6 @@ class handle_t { */ int get_size() const { return raft_handle_.get_comms().get_size(); } - /** - * @brief Get number of local gpus - * - * @return number of local gpus - */ - // FIXME: wrong for multi-node - int get_local_size() const { return raft_handle_.get_comms().get_size(); } - /** * @brief Get gpu rank * diff --git a/cpp/include/cugraph/mtmg/instance_manager.hpp b/cpp/include/cugraph/mtmg/instance_manager.hpp index 8bf62b56f4b..f819a5a0abe 100644 --- a/cpp/include/cugraph/mtmg/instance_manager.hpp +++ b/cpp/include/cugraph/mtmg/instance_manager.hpp @@ -18,7 +18,7 @@ #include -#include +#include #include @@ -37,16 +37,27 @@ class instance_manager_t { */ instance_manager_t(std::vector>&& handles, std::vector>&& nccl_comms, - std::vector&& device_ids, - int local_gpu_count) + std::vector&& device_ids) : thread_counter_{0}, raft_handle_{std::move(handles)}, nccl_comms_{std::move(nccl_comms)}, - device_ids_{std::move(device_ids)}, - local_gpu_count_{local_gpu_count} + device_ids_{std::move(device_ids)} { } + ~instance_manager_t() + { + int current_device{}; + RAFT_CUDA_TRY(cudaGetDevice(¤t_device)); + + for (size_t i = 0; i < nccl_comms_.size(); ++i) { + RAFT_CUDA_TRY(cudaSetDevice(device_ids_[i].value())); + RAFT_NCCL_TRY(ncclCommDestroy(*nccl_comms_[i])); + } + + RAFT_CUDA_TRY(cudaSetDevice(current_device)); + } + /** * @brief Get handle * @@ -54,18 +65,18 @@ class instance_manager_t { * the request. Threads will be assigned to GPUs in a round-robin fashion to * spread requesting threads around the GPU resources. * - * This function will be CPU thread-safe. + * This function is CPU thread-safe. * * @return a handle for this thread. */ handle_t get_handle() { - int local_id = thread_counter_++; + int local_id = thread_counter_++; + int gpu_id = local_id % raft_handle_.size(); + int thread_id = local_id / raft_handle_.size(); - RAFT_CUDA_TRY(cudaSetDevice(device_ids_[local_id % raft_handle_.size()].value())); - return handle_t(*raft_handle_[local_id % raft_handle_.size()], - local_id / raft_handle_.size(), - static_cast(local_id % raft_handle_.size())); + RAFT_CUDA_TRY(cudaSetDevice(device_ids_[gpu_id].value())); + return handle_t(*raft_handle_[gpu_id], thread_id, static_cast(gpu_id)); } /** @@ -79,7 +90,7 @@ class instance_manager_t { /** * @brief Number of local GPUs in the instance */ - int get_local_gpu_count() { return local_gpu_count_; } + int get_local_gpu_count() { return static_cast(raft_handle_.size()); } private: // FIXME: Should this be an std::map<> where the key is the rank? @@ -87,9 +98,11 @@ class instance_manager_t { // (or no) GPUs, so mapping rank to a handle might be a challenge // std::vector> raft_handle_{}; + + // FIXME: Explore what RAFT changes might be desired to allow the ncclComm_t + // to be managed by RAFT instead of cugraph::mtmg std::vector> nccl_comms_{}; std::vector device_ids_{}; - int local_gpu_count_{}; std::atomic thread_counter_{0}; }; diff --git a/cpp/include/cugraph/mtmg/resource_manager.hpp b/cpp/include/cugraph/mtmg/resource_manager.hpp index b4633626e7c..127944cf7ba 100644 --- a/cpp/include/cugraph/mtmg/resource_manager.hpp +++ b/cpp/include/cugraph/mtmg/resource_manager.hpp @@ -23,8 +23,8 @@ #include #include -#include #include +#include #include #include @@ -41,6 +41,11 @@ namespace mtmg { * register_local_gpu (or register_remote_gpu once we support a multi-node * configuration) to allocate resources that can be used in the mtmg space. * + * Each GPU in the cluster should be given a unique global rank, an integer + * that will be used to reference the GPU within the resource manager. It + * is recommended that the GPUs be numbered sequentially from 0, although this + * is not required. + * * When we want to execute some graph computations, we need to create an instance for execution. * Based on how big a subset of the desired compute resources is desired, we can allocate some * number of GPUs to the problem (up to the total set of managed resources). @@ -48,7 +53,7 @@ namespace mtmg { * The returned instance can be used to create a graph, execute one or more algorithms, etc. Once * we are done the caller can delete the instance. * - * At the moment, the caller is assumed to be responsible for scheduling use of the resources. + * The caller is assumed to be responsible for scheduling use of the resources. * * For our first release, we will only consider a single node multi-GPU configuration, so the remote * GPU methods are currently disabled via ifdef. @@ -63,25 +68,28 @@ class resource_manager_t { /** * @brief add a local GPU to the resource manager. * - * @param rank The rank to assign to the local GPU - * @param device_id The device_id corresponding to this rank + * @param global_rank The global rank to assign to the local GPU + * @param local_device_id The local device_id corresponding to this rank */ - void register_local_gpu(int rank, rmm::cuda_device_id device_id) + void register_local_gpu(int global_rank, rmm::cuda_device_id local_device_id) { std::lock_guard lock(lock_); - CUGRAPH_EXPECTS(local_rank_map_.find(rank) == local_rank_map_.end(), - "cannot register same rank multiple times"); + CUGRAPH_EXPECTS(remote_rank_set_.find(global_rank) == remote_rank_set_.end(), + "cannot register same global_rank as local and remote"); + CUGRAPH_EXPECTS(local_rank_map_.find(global_rank) == local_rank_map_.end(), + "cannot register same global_rank multiple times"); int num_gpus_this_node; RAFT_CUDA_TRY(cudaGetDeviceCount(&num_gpus_this_node)); - CUGRAPH_EXPECTS((device_id.value() >= 0) && (device_id.value() < num_gpus_this_node), - "device id out of range"); + CUGRAPH_EXPECTS( + (local_device_id.value() >= 0) && (local_device_id.value() < num_gpus_this_node), + "local device id out of range"); - local_rank_map_.insert(std::pair(rank, device_id)); + local_rank_map_.insert(std::pair(global_rank, local_device_id)); - RAFT_CUDA_TRY(cudaSetDevice(device_id.value())); + RAFT_CUDA_TRY(cudaSetDevice(local_device_id.value())); // FIXME: There is a bug in the cuda_memory_resource that results in a Hang. // using the pool resource as a work-around. @@ -89,23 +97,43 @@ class resource_manager_t { // There is a deprecated environment variable: NCCL_LAUNCH_MODE=GROUP // which should temporarily work around this problem. // + // Further NOTE: multi-node requires the NCCL_LAUNCH_MODE=GROUP feature + // to be enabled even with the pool memory resource. + // // Ultimately there should be some RMM parameters passed into this function // (or the constructor of the object) to configure this behavior #if 0 auto per_device_it = per_device_rmm_resources_.insert( - std::pair{rank, std::make_shared()}); + std::pair{global_rank, std::make_shared()}); #else auto const [free, total] = rmm::detail::available_device_memory(); auto const min_alloc = rmm::detail::align_down(std::min(free, total / 6), rmm::detail::CUDA_ALLOCATION_ALIGNMENT); auto per_device_it = per_device_rmm_resources_.insert( - std::pair{rank, + std::pair{global_rank, rmm::mr::make_owning_wrapper( std::make_shared(), min_alloc)}); #endif - rmm::mr::set_per_device_resource(device_id, per_device_it.first->second.get()); + rmm::mr::set_per_device_resource(local_device_id, per_device_it.first->second.get()); + } + + /** + * @brief add a remote GPU to the resource manager. + * + * @param global_rank The global rank to assign to the remote GPU + */ + void register_remote_gpu(int global_rank) + { + std::lock_guard lock(lock_); + + CUGRAPH_EXPECTS(local_rank_map_.find(global_rank) == local_rank_map_.end(), + "cannot register same global_rank as local and remote"); + CUGRAPH_EXPECTS(remote_rank_set_.find(global_rank) == remote_rank_set_.end(), + "cannot register same global_rank multiple times"); + + remote_rank_set_.insert(global_rank); } /** @@ -121,64 +149,76 @@ class resource_manager_t { * @param instance_manager_id a ncclUniqueId that is shared by all processes participating * in this instance. All processes must use the same ID in this call, it is up * to the calling code to share this ID properly before the call. + * @param n_streams The number of streams to create in a stream pool for + * each GPU. Defaults to 16. * * @return unique pointer to instance manager */ - std::unique_ptr create_instance_manager( - std::vector ranks_to_include, ncclUniqueId instance_manager_id) const + std::unique_ptr create_instance_manager(std::vector ranks_to_include, + ncclUniqueId instance_manager_id, + size_t n_streams = 16) const { - std::for_each( - ranks_to_include.begin(), ranks_to_include.end(), [local_ranks = local_rank_map_](int rank) { - CUGRAPH_EXPECTS(local_ranks.find(rank) != local_ranks.end(), - "requesting inclusion of an invalid rank"); - }); + std::vector local_ranks_to_include; + + std::copy_if(ranks_to_include.begin(), + ranks_to_include.end(), + std::back_inserter(local_ranks_to_include), + [&local_ranks = local_rank_map_](int rank) { + return (local_ranks.find(rank) != local_ranks.end()); + }); + // FIXME: Explore what RAFT changes might be desired to allow the ncclComm_t + // to be managed by RAFT instead of cugraph::mtmg std::vector> nccl_comms{}; std::vector> handles{}; std::vector device_ids{}; - nccl_comms.reserve(ranks_to_include.size()); - handles.reserve(ranks_to_include.size()); - device_ids.reserve(ranks_to_include.size()); + nccl_comms.reserve(local_ranks_to_include.size()); + handles.reserve(local_ranks_to_include.size()); + device_ids.reserve(local_ranks_to_include.size()); - // FIXME: not quite right for multi-node auto gpu_row_comm_size = static_cast(sqrt(static_cast(ranks_to_include.size()))); while (ranks_to_include.size() % gpu_row_comm_size != 0) { --gpu_row_comm_size; } - // FIXME: not quite right for multi-node - for (size_t i = 0; i < ranks_to_include.size(); ++i) { - int rank = ranks_to_include[i]; + int current_device{}; + RAFT_CUDA_TRY(cudaGetDevice(¤t_device)); + RAFT_NCCL_TRY(ncclGroupStart()); + + for (size_t i = 0; i < local_ranks_to_include.size(); ++i) { + int rank = local_ranks_to_include[i]; auto pos = local_rank_map_.find(rank); RAFT_CUDA_TRY(cudaSetDevice(pos->second.value())); - raft::handle_t tmp_handle; - nccl_comms.push_back(std::make_unique()); handles.push_back( - std::make_unique(tmp_handle, per_device_rmm_resources_.find(rank)->second)); + std::make_unique(rmm::cuda_stream_per_thread, + std::make_shared(n_streams), + per_device_rmm_resources_.find(rank)->second)); device_ids.push_back(pos->second); + + RAFT_NCCL_TRY( + ncclCommInitRank(nccl_comms[i].get(), ranks_to_include.size(), instance_manager_id, rank)); + raft::comms::build_comms_nccl_only( + handles[i].get(), *nccl_comms[i], ranks_to_include.size(), rank); } + RAFT_NCCL_TRY(ncclGroupEnd()); + RAFT_CUDA_TRY(cudaSetDevice(current_device)); std::vector running_threads; - for (size_t i = 0; i < ranks_to_include.size(); ++i) { + for (size_t i = 0; i < local_ranks_to_include.size(); ++i) { running_threads.emplace_back([instance_manager_id, idx = i, gpu_row_comm_size, comm_size = ranks_to_include.size(), - &ranks_to_include, - &local_rank_map = local_rank_map_, + &local_ranks_to_include, + &device_ids, &nccl_comms, &handles]() { - int rank = ranks_to_include[idx]; - auto pos = local_rank_map.find(rank); - RAFT_CUDA_TRY(cudaSetDevice(pos->second.value())); - - NCCL_TRY(ncclCommInitRank(nccl_comms[idx].get(), comm_size, instance_manager_id, rank)); - - raft::comms::build_comms_nccl_only(handles[idx].get(), *nccl_comms[idx], comm_size, rank); + int rank = local_ranks_to_include[idx]; + RAFT_CUDA_TRY(cudaSetDevice(device_ids[idx].value())); cugraph::partition_manager::init_subcomm(*handles[idx], gpu_row_comm_size); }); @@ -186,9 +226,8 @@ class resource_manager_t { std::for_each(running_threads.begin(), running_threads.end(), [](auto& t) { t.join(); }); - // FIXME: Update for multi-node return std::make_unique( - std::move(handles), std::move(nccl_comms), std::move(device_ids), ranks_to_include.size()); + std::move(handles), std::move(nccl_comms), std::move(device_ids)); } /** @@ -200,24 +239,24 @@ class resource_manager_t { { std::lock_guard lock(lock_); - // - // C++20 mechanism: - // return std::vector{ std::views::keys(local_rank_map_).begin(), - // std::views::keys(local_rank_map_).end() }; - // Would need a bit more complicated to handle remote_rank_map_ also - // - std::vector registered_ranks(local_rank_map_.size()); + std::vector registered_ranks(local_rank_map_.size() + remote_rank_set_.size()); std::transform( local_rank_map_.begin(), local_rank_map_.end(), registered_ranks.begin(), [](auto pair) { return pair.first; }); + std::copy(remote_rank_set_.begin(), + remote_rank_set_.end(), + registered_ranks.begin() + local_rank_map_.size()); + + std::sort(registered_ranks.begin(), registered_ranks.end()); return registered_ranks; } private: mutable std::mutex lock_{}; std::map local_rank_map_{}; + std::set remote_rank_set_{}; std::map> per_device_rmm_resources_{}; }; diff --git a/cpp/include/cugraph/mtmg/vertex_result_view.hpp b/cpp/include/cugraph/mtmg/vertex_result_view.hpp index 7a7070d6f2a..a349bb95333 100644 --- a/cpp/include/cugraph/mtmg/vertex_result_view.hpp +++ b/cpp/include/cugraph/mtmg/vertex_result_view.hpp @@ -21,6 +21,8 @@ #include #include +#include + namespace cugraph { namespace mtmg { diff --git a/cpp/include/cugraph/utilities/device_comm.hpp b/cpp/include/cugraph/utilities/device_comm.hpp index 7087724921a..990074e781b 100644 --- a/cpp/include/cugraph/utilities/device_comm.hpp +++ b/cpp/include/cugraph/utilities/device_comm.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -806,9 +806,6 @@ device_sendrecv(raft::comms::comms_t const& comm, size_t constexpr tuple_size = thrust::tuple_size::value_type>::value; - // FIXME: NCCL 2.7 supports only one ncclSend and one ncclRecv for a source rank and destination - // rank inside ncclGroupStart/ncclGroupEnd, so we cannot place this inside - // ncclGroupStart/ncclGroupEnd, this restriction will be lifted in NCCL 2.8 detail::device_sendrecv_tuple_iterator_element_impl::value_type>::value; - // FIXME: NCCL 2.7 supports only one ncclSend and one ncclRecv for a source rank and destination - // rank inside ncclGroupStart/ncclGroupEnd, so we cannot place this inside - // ncclGroupStart/ncclGroupEnd, this restriction will be lifted in NCCL 2.8 detail::device_multicast_sendrecv_tuple_iterator_element_impl -#include -#include +#include -#include -#include -#include -#include +#include #include -#include -#include namespace cugraph { @@ -44,12 +37,12 @@ struct pack_bool_t { __device__ uint32_t operator()(size_t i) const { - auto first = i * (sizeof(uint32_t) * 8); - auto last = std::min((i + 1) * (sizeof(uint32_t) * 8), num_bools); + auto first = i * packed_bools_per_word(); + auto last = std::min((i + 1) * packed_bools_per_word(), num_bools); uint32_t ret{0}; for (auto j = first; j < last; ++j) { if (*(bool_first + j)) { - auto mask = uint32_t{1} << (j % (sizeof(uint32_t) * 8)); + auto mask = packed_bool_mask(j); ret |= mask; } } @@ -57,6 +50,22 @@ struct pack_bool_t { } }; +template +struct check_bit_set_t { + PackedBoolIterator bitmap_first{}; + T idx_first{}; + + static_assert( + std::is_same_v::value_type, uint32_t>); + + __device__ bool operator()(T idx) const + { + auto offset = idx - idx_first; + return static_cast(*(bitmap_first + packed_bool_offset(offset)) & + packed_bool_mask(offset)); + } +}; + template struct indirection_t { Iterator first{}; @@ -80,7 +89,14 @@ struct indirection_if_idx_valid_t { }; template -struct not_equal_t { +struct is_equal_t { + T compare{}; + + __device__ bool operator()(T val) const { return val == compare; } +}; + +template +struct is_not_equal_t { T compare{}; __device__ bool operator()(T val) const { return val != compare; } @@ -96,19 +112,6 @@ struct is_first_in_run_t { } }; -template -struct check_bit_set_t { - uint32_t const* bitmaps{nullptr}; - T idx_first{}; - - __device__ bool operator()(T idx) const - { - auto offset = idx - idx_first; - auto mask = uint32_t{1} << (offset % (sizeof(uint32_t) * 8)); - return (*(bitmaps + (offset / (sizeof(uint32_t) * 8))) & mask) > uint32_t{0}; - } -}; - template struct check_in_range_t { T min{}; // inclusive diff --git a/cpp/include/cugraph/utilities/host_scalar_comm.hpp b/cpp/include/cugraph/utilities/host_scalar_comm.hpp index 6d3d772c2a2..4e6ec35b9d5 100644 --- a/cpp/include/cugraph/utilities/host_scalar_comm.hpp +++ b/cpp/include/cugraph/utilities/host_scalar_comm.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -254,19 +254,11 @@ template std::enable_if_t::value, std::vector> host_scalar_allgather( raft::comms::comms_t const& comm, T input, cudaStream_t stream) { - std::vector rx_counts(comm.get_size(), size_t{1}); - std::vector displacements(rx_counts.size(), size_t{0}); - std::iota(displacements.begin(), displacements.end(), size_t{0}); - rmm::device_uvector d_outputs(rx_counts.size(), stream); + rmm::device_uvector d_outputs(comm.get_size(), stream); raft::update_device(d_outputs.data() + comm.get_rank(), &input, 1, stream); - // FIXME: better use allgather - comm.allgatherv(d_outputs.data() + comm.get_rank(), - d_outputs.data(), - rx_counts.data(), - displacements.data(), - stream); - std::vector h_outputs(rx_counts.size()); - raft::update_host(h_outputs.data(), d_outputs.data(), rx_counts.size(), stream); + comm.allgather(d_outputs.data() + comm.get_rank(), d_outputs.data(), size_t{1}, stream); + std::vector h_outputs(d_outputs.size()); + raft::update_host(h_outputs.data(), d_outputs.data(), d_outputs.size(), stream); auto status = comm.sync_stream(stream); CUGRAPH_EXPECTS(status == raft::comms::status_t::SUCCESS, "sync_stream() failure."); return h_outputs; @@ -277,11 +269,6 @@ std::enable_if_t::value, std::vector::value; - std::vector rx_counts(comm.get_size(), tuple_size); - std::vector displacements(rx_counts.size(), size_t{0}); - for (size_t i = 0; i < displacements.size(); ++i) { - displacements[i] = i * tuple_size; - } std::vector h_tuple_scalar_elements(tuple_size); rmm::device_uvector d_allgathered_tuple_scalar_elements(comm.get_size() * tuple_size, stream); @@ -292,12 +279,10 @@ host_scalar_allgather(raft::comms::comms_t const& comm, T input, cudaStream_t st h_tuple_scalar_elements.data(), tuple_size, stream); - // FIXME: better use allgather - comm.allgatherv(d_allgathered_tuple_scalar_elements.data() + comm.get_rank() * tuple_size, - d_allgathered_tuple_scalar_elements.data(), - rx_counts.data(), - displacements.data(), - stream); + comm.allgather(d_allgathered_tuple_scalar_elements.data() + comm.get_rank() * tuple_size, + d_allgathered_tuple_scalar_elements.data(), + tuple_size, + stream); std::vector h_allgathered_tuple_scalar_elements(comm.get_size() * tuple_size); raft::update_host(h_allgathered_tuple_scalar_elements.data(), d_allgathered_tuple_scalar_elements.data(), @@ -318,6 +303,71 @@ host_scalar_allgather(raft::comms::comms_t const& comm, T input, cudaStream_t st return ret; } +template +std::enable_if_t::value, T> host_scalar_scatter( + raft::comms::comms_t const& comm, + std::vector const& inputs, // relevant only in root + int root, + cudaStream_t stream) +{ + CUGRAPH_EXPECTS( + ((comm.get_rank() == root) && (inputs.size() == static_cast(comm.get_size()))) || + ((comm.get_rank() != root) && (inputs.size() == 0)), + "inputs.size() should match with comm.get_size() in root and should be 0 otherwise."); + rmm::device_uvector d_outputs(comm.get_size(), stream); + if (comm.get_rank() == root) { + raft::update_device(d_outputs.data(), inputs.data(), inputs.size(), stream); + } + comm.bcast(d_outputs.data(), d_outputs.size(), root, stream); + T h_output{}; + raft::update_host(&h_output, d_outputs.data() + comm.get_rank(), 1, stream); + auto status = comm.sync_stream(stream); + CUGRAPH_EXPECTS(status == raft::comms::status_t::SUCCESS, "sync_stream() failure."); + return h_output; +} + +template +std::enable_if_t::value, T> host_scalar_scatter( + raft::comms::comms_t const& comm, + std::vector const& inputs, // relevant only in root + int root, + cudaStream_t stream) +{ + CUGRAPH_EXPECTS( + ((comm.get_rank() == root) && (inputs.size() == static_cast(comm.get_size()))) || + ((comm.get_rank() != root) && (inputs.size() == 0)), + "inputs.size() should match with comm.get_size() in root and should be 0 otherwise."); + size_t constexpr tuple_size = thrust::tuple_size::value; + rmm::device_uvector d_scatter_tuple_scalar_elements(comm.get_size() * tuple_size, + stream); + if (comm.get_rank() == root) { + for (int i = 0; i < comm.get_size(); ++i) { + std::vector h_tuple_scalar_elements(tuple_size); + detail::update_vector_of_tuple_scalar_elements_from_tuple_impl() + .update(h_tuple_scalar_elements, inputs[i]); + raft::update_device(d_scatter_tuple_scalar_elements.data() + i * tuple_size, + h_tuple_scalar_elements.data(), + tuple_size, + stream); + } + } + comm.bcast( + d_scatter_tuple_scalar_elements.data(), d_scatter_tuple_scalar_elements.size(), root, stream); + std::vector h_tuple_scalar_elements(tuple_size); + raft::update_host(h_tuple_scalar_elements.data(), + d_scatter_tuple_scalar_elements.data() + comm.get_rank() * tuple_size, + tuple_size, + stream); + auto status = comm.sync_stream(stream); + CUGRAPH_EXPECTS(status == raft::comms::status_t::SUCCESS, "sync_stream() failure."); + + T ret{}; + detail::update_tuple_from_vector_of_tuple_scalar_elements_impl().update( + ret, h_tuple_scalar_elements); + + return ret; +} + // Return value is valid only in root (return value may better be std::optional in C++17 or later) template std::enable_if_t::value, std::vector> host_scalar_gather( diff --git a/cpp/include/cugraph/utilities/mask_utils.cuh b/cpp/include/cugraph/utilities/mask_utils.cuh new file mode 100644 index 00000000000..ab1403d019b --- /dev/null +++ b/cpp/include/cugraph/utilities/mask_utils.cuh @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace cugraph { + +namespace detail { + +template // should be packed bool +__device__ size_t count_set_bits(MaskIterator mask_first, size_t start_offset, size_t num_bits) +{ + static_assert( + std::is_same_v::value_type, uint32_t>); + + size_t ret{0}; + + mask_first = mask_first + packed_bool_offset(start_offset); + start_offset = start_offset % packed_bools_per_word(); + if (start_offset != 0) { + auto mask = ~packed_bool_partial_mask(start_offset); + if (start_offset + num_bits < packed_bools_per_word()) { + mask &= packed_bool_partial_mask(start_offset + num_bits); + } + ret += __popc(*mask_first & mask); + num_bits -= __popc(mask); + ++mask_first; + } + + return thrust::transform_reduce( + thrust::seq, + thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(packed_bool_size(num_bits)), + [mask_first, num_bits] __device__(size_t i) { + auto word = *(mask_first + i); + if ((i + 1) * packed_bools_per_word() > num_bits) { + word &= packed_bool_partial_mask(num_bits % packed_bools_per_word()); + } + return static_cast(__popc(word)); + }, + ret, + thrust::plus{}); +} + +template ::value_type, // for packed bool support + typename output_value_type = typename thrust::iterator_traits< + OutputIterator>::value_type> // for packed bool support +__device__ size_t copy_if_mask_set(InputIterator input_first, + MaskIterator mask_first, + OutputIterator output_first, + size_t input_start_offset, + size_t output_start_offset, + size_t num_items) +{ + static_assert( + std::is_same_v::value_type, uint32_t>); + static_assert( + std::is_same_v::value_type, input_value_type> || + cugraph::has_packed_bool_element()); + static_assert(std::is_same_v::value_type, + output_value_type> || + cugraph::has_packed_bool_element()); + + static_assert(!cugraph::has_packed_bool_element() && + !cugraph::has_packed_bool_element(), + "unimplemented."); + + return static_cast(thrust::distance( + output_first + output_start_offset, + thrust::copy_if(thrust::seq, + input_first + input_start_offset, + input_first + (input_start_offset + num_items), + thrust::make_transform_iterator( + thrust::make_counting_iterator(size_t{0}), + check_bit_set_t{mask_first, size_t{0}}) + + input_start_offset, + output_first + output_start_offset, + is_equal_t{true}))); +} + +template // should be packed bool +size_t count_set_bits(raft::handle_t const& handle, MaskIterator mask_first, size_t num_bits) +{ + static_assert( + std::is_same_v::value_type, uint32_t>); + + return thrust::transform_reduce( + handle.get_thrust_policy(), + thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(packed_bool_size(num_bits)), + [mask_first, num_bits] __device__(size_t i) { + auto word = *(mask_first + i); + if ((i + 1) * packed_bools_per_word() > num_bits) { + word &= packed_bool_partial_mask(num_bits % packed_bools_per_word()); + } + return static_cast(__popc(word)); + }, + size_t{0}, + thrust::plus{}); +} + +template +OutputIterator copy_if_mask_set(raft::handle_t const& handle, + InputIterator input_first, + InputIterator input_last, + MaskIterator mask_first, + OutputIterator output_first) +{ + return thrust::copy_if( + handle.get_thrust_policy(), + input_first, + input_last, + thrust::make_transform_iterator(thrust::make_counting_iterator(size_t{0}), + check_bit_set_t{mask_first, size_t{0}}), + output_first, + is_equal_t{true}); +} + +} // namespace detail + +} // namespace cugraph diff --git a/cpp/include/cugraph/utilities/packed_bool_utils.hpp b/cpp/include/cugraph/utilities/packed_bool_utils.hpp index 9557b11e8e0..b418d5afc35 100644 --- a/cpp/include/cugraph/utilities/packed_bool_utils.hpp +++ b/cpp/include/cugraph/utilities/packed_bool_utils.hpp @@ -47,6 +47,13 @@ has_packed_bool_element(std::index_sequence) } // namespace detail +template +constexpr bool is_packed_bool() +{ + return std::is_same_v::value_type, uint32_t> && + std::is_same_v; +} + // sizeof(uint32_t) * 8 packed Boolean values are stored using one uint32_t template constexpr bool has_packed_bool_element() @@ -85,6 +92,13 @@ constexpr uint32_t packed_bool_mask(T bool_offset) constexpr uint32_t packed_bool_full_mask() { return uint32_t{0xffffffff}; } +template +constexpr uint32_t packed_bool_partial_mask(T num_set_bits) +{ + assert(static_cast(num_set_bits) <= sizeof(uint32_t) * 8); + return uint32_t{0xffffffff} >> (sizeof(uint32_t) * 8 - num_set_bits); +} + constexpr uint32_t packed_bool_empty_mask() { return uint32_t{0x0}; } template diff --git a/cpp/include/cugraph/utilities/shuffle_comm.cuh b/cpp/include/cugraph/utilities/shuffle_comm.cuh index 6a260144324..ab6a54cc1c0 100644 --- a/cpp/include/cugraph/utilities/shuffle_comm.cuh +++ b/cpp/include/cugraph/utilities/shuffle_comm.cuh @@ -80,7 +80,6 @@ compute_tx_rx_counts_offsets_ranks(raft::comms::comms_t const& comm, rmm::device_uvector d_rx_value_counts(comm_size, stream_view); - // FIXME: this needs to be replaced with AlltoAll once NCCL 2.8 is released. std::vector tx_counts(comm_size, size_t{1}); std::vector tx_offsets(comm_size); std::iota(tx_offsets.begin(), tx_offsets.end(), size_t{0}); @@ -835,7 +834,6 @@ auto shuffle_values(raft::comms::comms_t const& comm, allocate_dataframe_buffer::value_type>( rx_offsets.size() > 0 ? rx_offsets.back() + rx_counts.back() : size_t{0}, stream_view); - // FIXME: this needs to be replaced with AlltoAll once NCCL 2.8 is released // (if num_tx_dst_ranks == num_rx_src_ranks == comm_size). device_multicast_sendrecv(comm, tx_value_first, @@ -889,7 +887,6 @@ auto groupby_gpu_id_and_shuffle_values(raft::comms::comms_t const& comm, allocate_dataframe_buffer::value_type>( rx_offsets.size() > 0 ? rx_offsets.back() + rx_counts.back() : size_t{0}, stream_view); - // FIXME: this needs to be replaced with AlltoAll once NCCL 2.8 is released // (if num_tx_dst_ranks == num_rx_src_ranks == comm_size). device_multicast_sendrecv(comm, tx_value_first, @@ -946,7 +943,6 @@ auto groupby_gpu_id_and_shuffle_kv_pairs(raft::comms::comms_t const& comm, allocate_dataframe_buffer::value_type>( rx_keys.size(), stream_view); - // FIXME: this needs to be replaced with AlltoAll once NCCL 2.8 is released // (if num_tx_dst_ranks == num_rx_src_ranks == comm_size). device_multicast_sendrecv(comm, tx_key_first, @@ -959,7 +955,6 @@ auto groupby_gpu_id_and_shuffle_kv_pairs(raft::comms::comms_t const& comm, rx_src_ranks, stream_view); - // FIXME: this needs to be replaced with AlltoAll once NCCL 2.8 is released // (if num_tx_dst_ranks == num_rx_src_ranks == comm_size). device_multicast_sendrecv(comm, tx_value_first, diff --git a/cpp/include/cugraph_c/centrality_algorithms.h b/cpp/include/cugraph_c/centrality_algorithms.h index 0ac0e58540f..fb5d4b63b9c 100644 --- a/cpp/include/cugraph_c/centrality_algorithms.h +++ b/cpp/include/cugraph_c/centrality_algorithms.h @@ -23,8 +23,6 @@ #include /** @defgroup centrality Centrality algorithms - * @ingroup c_api - * @{ */ #ifdef __cplusplus @@ -39,7 +37,8 @@ typedef struct { } cugraph_centrality_result_t; /** - * @brief Get the vertex ids from the centrality result + * @ingroup centrality + * @brief Get the vertex ids from the centrality result * * @param [in] result The result from a centrality algorithm * @return type erased array of vertex ids @@ -48,7 +47,8 @@ cugraph_type_erased_device_array_view_t* cugraph_centrality_result_get_vertices( cugraph_centrality_result_t* result); /** - * @brief Get the centrality values from a centrality algorithm result + * @ingroup centrality + * @brief Get the centrality values from a centrality algorithm result * * @param [in] result The result from a centrality algorithm * @return type erased array view of centrality values @@ -57,6 +57,7 @@ cugraph_type_erased_device_array_view_t* cugraph_centrality_result_get_values( cugraph_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the number of iterations executed from the algorithm metadata * * @param [in] result The result from a centrality algorithm @@ -65,6 +66,7 @@ cugraph_type_erased_device_array_view_t* cugraph_centrality_result_get_values( size_t cugraph_centrality_result_get_num_iterations(cugraph_centrality_result_t* result); /** + * @ingroup centrality * @brief Returns true if the centrality algorithm converged * * @param [in] result The result from a centrality algorithm @@ -73,6 +75,7 @@ size_t cugraph_centrality_result_get_num_iterations(cugraph_centrality_result_t* bool_t cugraph_centrality_result_converged(cugraph_centrality_result_t* result); /** + * @ingroup centrality * @brief Free centrality result * * @param [in] result The result from a centrality algorithm @@ -409,6 +412,7 @@ typedef struct { } cugraph_edge_centrality_result_t; /** + * @ingroup centrality * @brief Get the src vertex ids from an edge centrality result * * @param [in] result The result from an edge centrality algorithm @@ -418,6 +422,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_src_ cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the dst vertex ids from an edge centrality result * * @param [in] result The result from an edge centrality algorithm @@ -427,6 +432,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_dst_ cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the edge ids from an edge centrality result * * @param [in] result The result from an edge centrality algorithm @@ -436,6 +442,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_edge cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the centrality values from an edge centrality algorithm result * * @param [in] result The result from an edge centrality algorithm @@ -445,6 +452,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_valu cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Free centrality result * * @param [in] result The result from a centrality algorithm @@ -491,6 +499,7 @@ typedef struct { } cugraph_hits_result_t; /** + * @ingroup centrality * @brief Get the vertex ids from the hits result * * @param [in] result The result from hits @@ -500,6 +509,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_vertices( cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the hubs values from the hits result * * @param [in] result The result from hits @@ -509,6 +519,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_hubs( cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the authorities values from the hits result * * @param [in] result The result from hits @@ -518,6 +529,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_authorities( cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the score differences between the last two iterations * * @param [in] result The result from hits @@ -526,6 +538,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_authorities( double cugraph_hits_result_get_hub_score_differences(cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the actual number of iterations * * @param [in] result The result from hits @@ -534,6 +547,7 @@ double cugraph_hits_result_get_hub_score_differences(cugraph_hits_result_t* resu size_t cugraph_hits_result_get_number_of_iterations(cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Free hits result * * @param [in] result The result from hits @@ -585,7 +599,3 @@ cugraph_error_code_t cugraph_hits( #ifdef __cplusplus } #endif - -/** - * @} - */ diff --git a/cpp/include/cugraph_c/community_algorithms.h b/cpp/include/cugraph_c/community_algorithms.h index 8f1015f8632..feab15c7eeb 100644 --- a/cpp/include/cugraph_c/community_algorithms.h +++ b/cpp/include/cugraph_c/community_algorithms.h @@ -23,7 +23,6 @@ #include /** @defgroup community Community algorithms - * @ingroup c_api * @{ */ diff --git a/cpp/include/cugraph_c/core_algorithms.h b/cpp/include/cugraph_c/core_algorithms.h index c0e348c3cf4..6db3269f61e 100644 --- a/cpp/include/cugraph_c/core_algorithms.h +++ b/cpp/include/cugraph_c/core_algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,9 @@ #include #include +/** @defgroup core Core algorithms + */ + #ifdef __cplusplus extern "C" { #endif @@ -40,6 +43,7 @@ typedef struct { } cugraph_k_core_result_t; /** + * @ingroup core * @brief Create a core_number result (in case it was previously extracted) * * @param [in] handle Handle for accessing resources @@ -58,6 +62,7 @@ cugraph_error_code_t cugraph_core_result_create( cugraph_error_t** error); /** + * @ingroup core * @brief Get the vertex ids from the core result * * @param [in] result The result from core number @@ -67,6 +72,7 @@ cugraph_type_erased_device_array_view_t* cugraph_core_result_get_vertices( cugraph_core_result_t* result); /** + * @ingroup core * @brief Get the core numbers from the core result * * @param [in] result The result from core number @@ -76,6 +82,7 @@ cugraph_type_erased_device_array_view_t* cugraph_core_result_get_core_numbers( cugraph_core_result_t* result); /** + * @ingroup core * @brief Free core result * * @param [in] result The result from core number @@ -83,6 +90,7 @@ cugraph_type_erased_device_array_view_t* cugraph_core_result_get_core_numbers( void cugraph_core_result_free(cugraph_core_result_t* result); /** + * @ingroup core * @brief Get the src vertex ids from the k-core result * * @param [in] result The result from k-core @@ -92,6 +100,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_src_vertices( cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Get the dst vertex ids from the k-core result * * @param [in] result The result from k-core @@ -101,6 +110,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_dst_vertices( cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Get the weights from the k-core result * * Returns NULL if the graph is unweighted @@ -112,6 +122,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_weights( cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Free k-core result * * @param [in] result The result from k-core @@ -119,6 +130,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_weights( void cugraph_k_core_result_free(cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Enumeration for computing core number */ typedef enum { diff --git a/cpp/include/cugraph_c/graph.h b/cpp/include/cugraph_c/graph.h index e910d8b1244..00fce0493a3 100644 --- a/cpp/include/cugraph_c/graph.h +++ b/cpp/include/cugraph_c/graph.h @@ -35,10 +35,11 @@ typedef struct { bool_t is_multigraph; } cugraph_graph_properties_t; -// FIXME: Add support for specifying isolated vertices /** * @brief Construct an SG graph * + * @deprecated This API will be deleted, use cugraph_graph_create_sg instead + * * @param [in] handle Handle for accessing resources * @param [in] properties Properties of the constructed graph * @param [in] src Device array containing the source vertex ids. @@ -51,11 +52,11 @@ typedef struct { argument that can be NULL if edge types are not used. * @param [in] store_transposed If true create the graph initially in transposed format * @param [in] renumber If true, renumber vertices to make an efficient data structure. - * If false, do not renumber. Renumbering is required if the vertices are not sequential - * integer values from 0 to num_vertices. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. * @param [in] do_expensive_check If true, do expensive checks to validate the input data * is consistent with software assumptions. If false bypass these checks. - * @param [in] properties Properties of the graph * @param [out] graph A pointer to the graph object * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS @@ -76,9 +77,64 @@ cugraph_error_code_t cugraph_sg_graph_create( cugraph_graph_t** graph, cugraph_error_t** error); +/** + * @brief Construct an SG graph + * + * @param [in] handle Handle for accessing resources + * @param [in] properties Properties of the constructed graph + * @param [in] vertices Optional device array containing a list of vertex ids + * (specify NULL if we should create vertex ids from the + * unique contents of @p src and @p dst) + * @param [in] src Device array containing the source vertex ids. + * @param [in] dst Device array containing the destination vertex ids + * @param [in] weights Device array containing the edge weights. Note that an unweighted + * graph can be created by passing weights == NULL. + * @param [in] edge_ids Device array containing the edge ids for each edge. Optional + argument that can be NULL if edge ids are not used. + * @param [in] edge_type_ids Device array containing the edge types for each edge. Optional + argument that can be NULL if edge types are not used. + * @param [in] store_transposed If true create the graph initially in transposed format + * @param [in] renumber If true, renumber vertices to make an efficient data structure. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. + * @param [in] drop_self_loops If true, drop any self loops that exist in the provided edge list. + * @param [in] drop_multi_edges If true, drop any multi edges that exist in the provided edge list. + * Note that setting this flag will arbitrarily select one instance of a multi edge to be the + * edge that survives. If the edges have properties that should be honored (e.g. sum the + weights, + * or take the maximum weight), the caller should remove specific edges themselves and not rely + * on this flag. + * @param [in] do_expensive_check If true, do expensive checks to validate the input data + * is consistent with software assumptions. If false bypass these checks. + * @param [out] graph A pointer to the graph object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * + * @return error code + */ +cugraph_error_code_t cugraph_graph_create_sg( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* vertices, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error); + /** * @brief Construct an SG graph from a CSR input * + * @deprecated This API will be deleted, use cugraph_graph_create_sg_from_csr instead + * * @param [in] handle Handle for accessing resources * @param [in] properties Properties of the constructed graph * @param [in] offsets Device array containing the CSR offsets array @@ -91,11 +147,11 @@ cugraph_error_code_t cugraph_sg_graph_create( argument that can be NULL if edge types are not used. * @param [in] store_transposed If true create the graph initially in transposed format * @param [in] renumber If true, renumber vertices to make an efficient data structure. - * If false, do not renumber. Renumbering is required if the vertices are not sequential - * integer values from 0 to num_vertices. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. * @param [in] do_expensive_check If true, do expensive checks to validate the input data * is consistent with software assumptions. If false bypass these checks. - * @param [in] properties Properties of the graph * @param [out] graph A pointer to the graph object * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS @@ -117,18 +173,50 @@ cugraph_error_code_t cugraph_sg_graph_create_from_csr( cugraph_error_t** error); /** - * @brief Destroy an SG graph + * @brief Construct an SG graph from a CSR input * - * @param [in] graph A pointer to the graph object to destroy + * @param [in] handle Handle for accessing resources + * @param [in] properties Properties of the constructed graph + * @param [in] offsets Device array containing the CSR offsets array + * @param [in] indices Device array containing the destination vertex ids + * @param [in] weights Device array containing the edge weights. Note that an unweighted + * graph can be created by passing weights == NULL. + * @param [in] edge_ids Device array containing the edge ids for each edge. Optional + argument that can be NULL if edge ids are not used. + * @param [in] edge_type_ids Device array containing the edge types for each edge. Optional + argument that can be NULL if edge types are not used. + * @param [in] store_transposed If true create the graph initially in transposed format + * @param [in] renumber If true, renumber vertices to make an efficient data structure. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. + * @param [in] do_expensive_check If true, do expensive checks to validate the input data + * is consistent with software assumptions. If false bypass these checks. + * @param [out] graph A pointer to the graph object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * + * @return error code */ -// FIXME: This should probably just be cugraph_graph_free -// but didn't want to confuse with original cugraph_free_graph -void cugraph_sg_graph_free(cugraph_graph_t* graph); +cugraph_error_code_t cugraph_graph_create_sg_from_csr( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* offsets, + const cugraph_type_erased_device_array_view_t* indices, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error); -// FIXME: Add support for specifying isolated vertices /** * @brief Construct an MG graph * + * @deprecated This API will be deleted, use cugraph_graph_create_mg instead + * * @param [in] handle Handle for accessing resources * @param [in] properties Properties of the constructed graph * @param [in] src Device array containing the source vertex ids @@ -165,13 +253,89 @@ cugraph_error_code_t cugraph_mg_graph_create( cugraph_graph_t** graph, cugraph_error_t** error); +/** + * @brief Construct an MG graph + * + * @param [in] handle Handle for accessing resources + * @param [in] properties Properties of the constructed graph + * @param [in] vertices List of device arrays containing the unique vertex ids. + * If NULL we will construct this internally using the unique + * entries specified in src and dst + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] src List of device array containing the source vertex ids + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] dst List of device array containing the destination vertex ids + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] weights List of device array containing the edge weights. Note that an + * unweighted graph can be created by passing weights == NULL. If a weighted graph is to be + * created, the weights device array should be created on each rank, but the pointer can be NULL and + * the size 0 if there are no inputs provided by this rank All entries in this list will be + * concatenated on this GPU into a single array. + * @param [in] edge_ids List of device array containing the edge ids for each edge. Optional + * argument that can be NULL if edge ids are not used. + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] edge_type_ids List of device array containing the edge types for each edge. + * Optional argument that can be NULL if edge types are not used. All entries in this list will be + * concatenated on this GPU into a single array. + * @param [in] store_transposed If true create the graph initially in transposed format + * @param [in] num_arrays The number of arrays specified in @p vertices, @p src, @p dst, @p + * weights, @p edge_ids and @p edge_type_ids + * @param [in] drop_self_loops If true, drop any self loops that exist in the provided edge list. + * @param [in] drop_multi_edges If true, drop any multi edges that exist in the provided edge list. + * Note that setting this flag will arbitrarily select one instance of a multi edge to be the + * edge that survives. If the edges have properties that should be honored (e.g. sum the + * weights, or take the maximum weight), the caller should do that on not rely on this flag. + * @param [in] do_expensive_check If true, do expensive checks to validate the input data + * is consistent with software assumptions. If false bypass these checks. + * @param [out] graph A pointer to the graph object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_graph_create_mg( + cugraph_resource_handle_t const* handle, + cugraph_graph_properties_t const* properties, + cugraph_type_erased_device_array_view_t const* const* vertices, + cugraph_type_erased_device_array_view_t const* const* src, + cugraph_type_erased_device_array_view_t const* const* dst, + cugraph_type_erased_device_array_view_t const* const* weights, + cugraph_type_erased_device_array_view_t const* const* edge_ids, + cugraph_type_erased_device_array_view_t const* const* edge_type_ids, + bool_t store_transposed, + size_t num_arrays, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error); + +/** + * @brief Destroy an graph + * + * @param [in] graph A pointer to the graph object to destroy + */ +void cugraph_graph_free(cugraph_graph_t* graph); + +/** + * @brief Destroy an SG graph + * + * @deprecated This API will be deleted, use cugraph_graph_free instead + * + * @param [in] graph A pointer to the graph object to destroy + */ +void cugraph_sg_graph_free(cugraph_graph_t* graph); + /** * @brief Destroy an MG graph * + * @deprecated This API will be deleted, use cugraph_graph_free instead + * * @param [in] graph A pointer to the graph object to destroy */ -// FIXME: This should probably just be cugraph_graph_free -// but didn't want to confuse with original cugraph_free_graph void cugraph_mg_graph_free(cugraph_graph_t* graph); /** diff --git a/cpp/include/cugraph_c/graph_functions.h b/cpp/include/cugraph_c/graph_functions.h index 655324df284..19b69922fa5 100644 --- a/cpp/include/cugraph_c/graph_functions.h +++ b/cpp/include/cugraph_c/graph_functions.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -136,6 +136,24 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_destinatio cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_weights( cugraph_induced_subgraph_result_t* induced_subgraph); +/** + * @brief Get the edge ids + * + * @param [in] induced_subgraph Opaque pointer to induced subgraph + * @return type erased array view of edge ids + */ +cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_result_t* induced_subgraph); + +/** + * @brief Get the edge types + * + * @param [in] induced_subgraph Opaque pointer to induced subgraph + * @return type erased array view of edge types + */ +cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_result_t* induced_subgraph); + /** * @brief Get the subgraph offsets * @@ -184,6 +202,33 @@ cugraph_error_code_t cugraph_extract_induced_subgraph( cugraph_induced_subgraph_result_t** result, cugraph_error_t** error); +// FIXME: Rename the return type +/** + * @brief Gather edgelist + * + * This function collects the edgelist from all ranks and stores the combine edgelist + * in each rank + * + * @param [in] handle Handle for accessing resources. + * @param [in] src Device array containing the source vertex ids. + * @param [in] dst Device array containing the destination vertex ids + * @param [in] weights Optional device array containing the edge weights + * @param [in] edge_ids Optional device array containing the edge ids for each edge. + * @param [in] edge_type_ids Optional device array containing the edge types for each edge + * @param [out] result Opaque pointer to gathered edgelist result + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_allgather(const cugraph_resource_handle_t* handle, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + cugraph_induced_subgraph_result_t** result, + cugraph_error_t** error); + #ifdef __cplusplus } #endif diff --git a/cpp/include/cugraph_c/labeling_algorithms.h b/cpp/include/cugraph_c/labeling_algorithms.h index f3e634dafe6..53dcc0d9419 100644 --- a/cpp/include/cugraph_c/labeling_algorithms.h +++ b/cpp/include/cugraph_c/labeling_algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ extern "C" { #endif /** @defgroup labeling Labeling algorithms - * @ingroup c_api - * @{ */ /** @@ -37,6 +35,7 @@ typedef struct { } cugraph_labeling_result_t; /** + * @ingroup labeling * @brief Get the vertex ids from the labeling result * * @param [in] result The result from a labeling algorithm @@ -46,6 +45,7 @@ cugraph_type_erased_device_array_view_t* cugraph_labeling_result_get_vertices( cugraph_labeling_result_t* result); /** + * @ingroup labeling * @brief Get the label values from the labeling result * * @param [in] result The result from a labeling algorithm @@ -55,6 +55,7 @@ cugraph_type_erased_device_array_view_t* cugraph_labeling_result_get_labels( cugraph_labeling_result_t* result); /** + * @ingroup labeling * @brief Free labeling result * * @param [in] result The result from a labeling algorithm @@ -104,7 +105,3 @@ cugraph_error_code_t cugraph_strongly_connected_components(const cugraph_resourc #ifdef __cplusplus } #endif - -/** - * @} - */ diff --git a/cpp/include/cugraph_c/resource_handle.h b/cpp/include/cugraph_c/resource_handle.h index a239c24afe9..0e45102aae2 100644 --- a/cpp/include/cugraph_c/resource_handle.h +++ b/cpp/include/cugraph_c/resource_handle.h @@ -57,6 +57,18 @@ typedef struct cugraph_resource_handle_ { */ cugraph_resource_handle_t* cugraph_create_resource_handle(void* raft_handle); +/** + * @brief get comm_size from resource handle + * + * If the resource handle has been configured for multi-gpu, this will return + * the comm_size for this cluster. If the resource handle has not been configured for + * multi-gpu this will always return 1. + * + * @param [in] handle Handle for accessing resources + * @return comm_size + */ +int cugraph_resource_handle_get_comm_size(const cugraph_resource_handle_t* handle); + /** * @brief get rank from resource handle * diff --git a/cpp/include/cugraph_c/sampling_algorithms.h b/cpp/include/cugraph_c/sampling_algorithms.h index 92fe50ef622..782bb5a3790 100644 --- a/cpp/include/cugraph_c/sampling_algorithms.h +++ b/cpp/include/cugraph_c/sampling_algorithms.h @@ -21,8 +21,7 @@ #include #include -/** @defgroup sampling Sampling algorithms - * @ingroup c_api +/** @defgroup samplingC Sampling algorithms * @{ */ diff --git a/cpp/include/cugraph_c/similarity_algorithms.h b/cpp/include/cugraph_c/similarity_algorithms.h index 1417d8ac566..b8f61b46545 100644 --- a/cpp/include/cugraph_c/similarity_algorithms.h +++ b/cpp/include/cugraph_c/similarity_algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,9 @@ #include #include +/** @defgroup similarity Similarity algorithms + */ + #ifdef __cplusplus extern "C" { #endif @@ -34,6 +37,7 @@ typedef struct { } cugraph_similarity_result_t; /** + * @ingroup similarity * @brief Get the similarity coefficient array * * @param [in] result The result from a similarity algorithm @@ -43,6 +47,7 @@ cugraph_type_erased_device_array_view_t* cugraph_similarity_result_get_similarit cugraph_similarity_result_t* result); /** + * @ingroup similarity * @brief Free similarity result * * @param [in] result The result from a similarity algorithm diff --git a/cpp/libcugraph_etl/CMakeLists.txt b/cpp/libcugraph_etl/CMakeLists.txt index 18271871087..ac0cb6959e8 100644 --- a/cpp/libcugraph_etl/CMakeLists.txt +++ b/cpp/libcugraph_etl/CMakeLists.txt @@ -25,7 +25,7 @@ include(rapids-find) rapids_cuda_init_architectures(CUGRAPH_ETL) -project(CUGRAPH_ETL VERSION 23.10.00 LANGUAGES C CXX CUDA) +project(CUGRAPH_ETL VERSION 23.12.00 LANGUAGES C CXX CUDA) if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0) diff --git a/cpp/src/c_api/allgather.cpp b/cpp/src/c_api/allgather.cpp new file mode 100644 index 00000000000..7ef401aa6b7 --- /dev/null +++ b/cpp/src/c_api/allgather.cpp @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { + +struct create_allgather_functor : public cugraph::c_api::abstract_functor { + raft::handle_t const& handle_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* src_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids_; + cugraph::c_api::cugraph_induced_subgraph_result_t* result_{}; + + create_allgather_functor( + raft::handle_t const& handle, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* src, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids) + : abstract_functor(), + handle_(handle), + src_(src), + dst_(dst), + weights_(weights), + edge_ids_(edge_ids), + edge_type_ids_(edge_type_ids) + { + } + + template + void operator()() + { + std::optional> edgelist_srcs{std::nullopt}; + if (src_) { + edgelist_srcs = rmm::device_uvector(src_->size_, handle_.get_stream()); + raft::copy( + edgelist_srcs->data(), src_->as_type(), src_->size_, handle_.get_stream()); + } + + std::optional> edgelist_dsts{std::nullopt}; + if (dst_) { + edgelist_dsts = rmm::device_uvector(dst_->size_, handle_.get_stream()); + raft::copy( + edgelist_dsts->data(), dst_->as_type(), dst_->size_, handle_.get_stream()); + } + + std::optional> edgelist_weights{std::nullopt}; + if (weights_) { + edgelist_weights = rmm::device_uvector(weights_->size_, handle_.get_stream()); + raft::copy(edgelist_weights->data(), + weights_->as_type(), + weights_->size_, + handle_.get_stream()); + } + + std::optional> edgelist_ids{std::nullopt}; + if (edge_ids_) { + edgelist_ids = rmm::device_uvector(edge_ids_->size_, handle_.get_stream()); + raft::copy( + edgelist_ids->data(), edge_ids_->as_type(), edge_ids_->size_, handle_.get_stream()); + } + + std::optional> edgelist_type_ids{std::nullopt}; + if (edge_type_ids_) { + edgelist_type_ids = + rmm::device_uvector(edge_type_ids_->size_, handle_.get_stream()); + raft::copy(edgelist_type_ids->data(), + edge_type_ids_->as_type(), + edge_type_ids_->size_, + handle_.get_stream()); + } + + auto& comm = handle_.get_comms(); + + if (edgelist_srcs) { + edgelist_srcs = cugraph::detail::device_allgatherv( + handle_, + comm, + raft::device_span(edgelist_srcs->data(), edgelist_srcs->size())); + } + + if (edgelist_dsts) { + edgelist_dsts = cugraph::detail::device_allgatherv( + handle_, + comm, + raft::device_span(edgelist_dsts->data(), edgelist_dsts->size())); + } + + rmm::device_uvector edge_offsets(2, handle_.get_stream()); + + std::vector h_edge_offsets{ + {0, edgelist_srcs ? edgelist_srcs->size() : edgelist_weights->size()}}; + raft::update_device( + edge_offsets.data(), h_edge_offsets.data(), h_edge_offsets.size(), handle_.get_stream()); + + cugraph::c_api::cugraph_induced_subgraph_result_t* result = NULL; + + if (edgelist_weights) { + edgelist_weights = cugraph::detail::device_allgatherv( + handle_, + comm, + raft::device_span(edgelist_weights->data(), edgelist_weights->size())); + } + + if (edgelist_ids) { + edgelist_ids = cugraph::detail::device_allgatherv( + handle_, comm, raft::device_span(edgelist_ids->data(), edgelist_ids->size())); + } + + if (edgelist_type_ids) { + edgelist_type_ids = + cugraph::detail::device_allgatherv(handle_, + comm, + raft::device_span( + edgelist_type_ids->data(), edgelist_type_ids->size())); + } + + result = new cugraph::c_api::cugraph_induced_subgraph_result_t{ + edgelist_srcs + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_srcs, src_->type_) + : NULL, + edgelist_dsts + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_dsts, dst_->type_) + : NULL, + edgelist_weights + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_weights, weights_->type_) + : NULL, + edgelist_ids + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_ids, edge_ids_->type_) + : NULL, + edgelist_type_ids ? new cugraph::c_api::cugraph_type_erased_device_array_t( + *edgelist_type_ids, edge_type_ids_->type_) + : NULL, + new cugraph::c_api::cugraph_type_erased_device_array_t(edge_offsets, + cugraph_data_type_id_t::SIZE_T)}; + + result_ = reinterpret_cast(result); + } +}; + +} // namespace + +extern "C" cugraph_error_code_t cugraph_allgather( + const cugraph_resource_handle_t* handle, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + cugraph_induced_subgraph_result_t** edgelist, + cugraph_error_t** error) +{ + *edgelist = nullptr; + *error = nullptr; + + auto p_handle = reinterpret_cast(handle); + auto p_src = + reinterpret_cast(src); + auto p_dst = + reinterpret_cast(dst); + auto p_weights = + reinterpret_cast(weights); + + auto p_edge_ids = + reinterpret_cast(edge_ids); + + auto p_edge_type_ids = + reinterpret_cast(edge_type_ids); + + CAPI_EXPECTS((dst == nullptr) || (src == nullptr) || p_src->size_ == p_dst->size_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != dst size.", + *error); + CAPI_EXPECTS((dst == nullptr) || (src == nullptr) || p_src->type_ == p_dst->type_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != dst type.", + *error); + + CAPI_EXPECTS((weights == nullptr) || (src == nullptr) || (p_weights->size_ == p_src->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != weights size.", + *error); + + cugraph_data_type_id_t vertex_type; + cugraph_data_type_id_t edge_type; + cugraph_data_type_id_t weight_type; + cugraph_data_type_id_t edge_type_id_type; + + if (src != nullptr) { + vertex_type = p_src->type_; + } else { + vertex_type = cugraph_data_type_id_t::INT32; + } + + if (weights != nullptr) { + weight_type = p_weights->type_; + } else { + weight_type = cugraph_data_type_id_t::FLOAT32; + } + + if (edge_ids != nullptr) { + edge_type = p_edge_ids->type_; + } else { + edge_type = cugraph_data_type_id_t::INT32; + } + + if (edge_type_ids != nullptr) { + edge_type_id_type = p_edge_type_ids->type_; + } else { + edge_type_id_type = cugraph_data_type_id_t::INT32; + } + + if (src != nullptr) { + CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids->size_ == p_src->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge id prop size", + *error); + + CAPI_EXPECTS((edge_type_ids == nullptr) || (p_edge_type_ids->size_ == p_src->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge type prop size", + *error); + } + + constexpr bool multi_gpu = false; + constexpr bool store_transposed = false; + + ::create_allgather_functor functor( + *p_handle->handle_, p_src, p_dst, p_weights, p_edge_ids, p_edge_type_ids); + + try { + cugraph::c_api::vertex_dispatcher( + vertex_type, edge_type, weight_type, edge_type_id_type, store_transposed, multi_gpu, functor); + + if (functor.error_code_ != CUGRAPH_SUCCESS) { + *error = reinterpret_cast(functor.error_.release()); + return functor.error_code_; + } + + *edgelist = reinterpret_cast(functor.result_); + } catch (std::exception const& ex) { + *error = reinterpret_cast(new cugraph::c_api::cugraph_error_t{ex.what()}); + return CUGRAPH_UNKNOWN_ERROR; + } + + return CUGRAPH_SUCCESS; +} diff --git a/cpp/src/c_api/capi_helper.cu b/cpp/src/c_api/capi_helper.cu index af0163b0512..0ee49f87265 100644 --- a/cpp/src/c_api/capi_helper.cu +++ b/cpp/src/c_api/capi_helper.cu @@ -44,7 +44,7 @@ shuffle_vertex_ids_and_offsets(raft::handle_t const& handle, thrust::make_zip_iterator(ids.end(), vertices.end())); auto return_offsets = cugraph::detail::compute_sparse_offsets( - ids.begin(), ids.end(), size_t{0}, size_t{offsets.size() - 1}, handle.get_stream()); + ids.begin(), ids.end(), size_t{0}, size_t{offsets.size() - 1}, true, handle.get_stream()); return std::make_tuple(std::move(vertices), std::move(return_offsets)); } diff --git a/cpp/src/c_api/extract_ego.cpp b/cpp/src/c_api/extract_ego.cpp index 8f510b79023..931d58b5185 100644 --- a/cpp/src/c_api/extract_ego.cpp +++ b/cpp/src/c_api/extract_ego.cpp @@ -135,6 +135,8 @@ struct extract_ego_functor : public cugraph::c_api::abstract_functor { new cugraph::c_api::cugraph_type_erased_device_array_t(dst, graph_->vertex_type_), wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*wgt, graph_->weight_type_) : NULL, + NULL, + NULL, new cugraph::c_api::cugraph_type_erased_device_array_t(edge_offsets, cugraph_data_type_id_t::SIZE_T)}; } diff --git a/cpp/src/c_api/graph_mg.cpp b/cpp/src/c_api/graph_mg.cpp index f50c7c08fb6..326022a3fa9 100644 --- a/cpp/src/c_api/graph_mg.cpp +++ b/cpp/src/c_api/graph_mg.cpp @@ -31,40 +31,85 @@ namespace { +template +rmm::device_uvector concatenate( + raft::handle_t const& handle, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* values, + size_t num_arrays) +{ + size_t num_values = std::transform_reduce( + values, values + num_arrays, size_t{0}, std::plus{}, [](auto p) { return p->size_; }); + + rmm::device_uvector results(num_values, handle.get_stream()); + size_t concat_pos{0}; + + for (size_t i = 0; i < num_arrays; ++i) { + raft::copy(results.data() + concat_pos, + values[i]->as_type(), + values[i]->size_, + handle.get_stream()); + concat_pos += values[i]->size_; + } + + return results; +} + struct create_graph_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; cugraph_graph_properties_t const* properties_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* src_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids_; - bool_t renumber_; - bool_t check_; + cugraph_data_type_id_t vertex_type_; cugraph_data_type_id_t edge_type_; + cugraph_data_type_id_t weight_type_; + cugraph_data_type_id_t edge_type_id_type_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* vertices_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* src_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* dst_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* weights_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_ids_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_type_ids_; + size_t num_arrays_; + bool_t renumber_; + bool_t drop_self_loops_; + bool_t drop_multi_edges_; + bool_t do_expensive_check_; cugraph::c_api::cugraph_graph_t* result_{}; - create_graph_functor(raft::handle_t const& handle, - cugraph_graph_properties_t const* properties, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* src, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids, - bool_t renumber, - bool_t check, - cugraph_data_type_id_t edge_type) + create_graph_functor( + raft::handle_t const& handle, + cugraph_graph_properties_t const* properties, + cugraph_data_type_id_t vertex_type, + cugraph_data_type_id_t edge_type, + cugraph_data_type_id_t weight_type, + cugraph_data_type_id_t edge_type_id_type, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* vertices, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* src, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* dst, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* weights, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_ids, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_type_ids, + size_t num_arrays, + bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check) : abstract_functor(), properties_(properties), + vertex_type_(vertex_type), + edge_type_(edge_type), + weight_type_(weight_type), + edge_type_id_type_(edge_type_id_type), handle_(handle), + vertices_(vertices), src_(src), dst_(dst), weights_(weights), edge_ids_(edge_ids), edge_type_ids_(edge_type_ids), + num_arrays_(num_arrays), renumber_(renumber), - check_(check), - edge_type_(edge_type) + drop_self_loops_(drop_self_loops), + drop_multi_edges_(drop_multi_edges), + do_expensive_check_(do_expensive_check) { } @@ -96,49 +141,27 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { edge_type_id_t>> new_edge_types{std::nullopt}; - rmm::device_uvector edgelist_srcs(src_->size_, handle_.get_stream()); - rmm::device_uvector edgelist_dsts(dst_->size_, handle_.get_stream()); + std::optional> vertex_list = + vertices_ ? std::make_optional(concatenate(handle_, vertices_, num_arrays_)) + : std::nullopt; - raft::copy( - edgelist_srcs.data(), src_->as_type(), src_->size_, handle_.get_stream()); - raft::copy( - edgelist_dsts.data(), dst_->as_type(), dst_->size_, handle_.get_stream()); + rmm::device_uvector edgelist_srcs = + concatenate(handle_, src_, num_arrays_); + rmm::device_uvector edgelist_dsts = + concatenate(handle_, dst_, num_arrays_); std::optional> edgelist_weights = - weights_ - ? std::make_optional(rmm::device_uvector(weights_->size_, handle_.get_stream())) - : std::nullopt; - - if (edgelist_weights) { - raft::copy(edgelist_weights->data(), - weights_->as_type(), - weights_->size_, - handle_.get_stream()); - } + weights_ ? std::make_optional(concatenate(handle_, weights_, num_arrays_)) + : std::nullopt; std::optional> edgelist_edge_ids = - edge_ids_ - ? std::make_optional(rmm::device_uvector(edge_ids_->size_, handle_.get_stream())) - : std::nullopt; - - if (edgelist_edge_ids) { - raft::copy(edgelist_edge_ids->data(), - edge_ids_->as_type(), - edge_ids_->size_, - handle_.get_stream()); - } + edge_ids_ ? std::make_optional(concatenate(handle_, edge_ids_, num_arrays_)) + : std::nullopt; std::optional> edgelist_edge_types = - edge_type_ids_ ? std::make_optional(rmm::device_uvector( - edge_type_ids_->size_, handle_.get_stream())) - : std::nullopt; - - if (edgelist_edge_types) { - raft::copy(edgelist_edge_types->data(), - edge_type_ids_->as_type(), - edge_type_ids_->size_, - handle_.get_stream()); - } + edge_type_ids_ + ? std::make_optional(concatenate(handle_, edge_type_ids_, num_arrays_)) + : std::nullopt; std::tie(store_transposed ? edgelist_dsts : edgelist_srcs, store_transposed ? edgelist_srcs : edgelist_dsts, @@ -153,6 +176,11 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { std::move(edgelist_edge_ids), std::move(edgelist_edge_types)); + if (vertex_list) { + vertex_list = cugraph::detail::shuffle_ext_vertices_to_local_gpu_by_vertex_partitioning( + handle_, std::move(*vertex_list)); + } + auto graph = new cugraph::graph_t(handle_); rmm::device_uvector* number_map = @@ -170,6 +198,28 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { cugraph::graph_view_t, edge_type_id_t>(handle_); + if (drop_self_loops_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_self_loops(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + + if (drop_multi_edges_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_multi_edges(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + std::tie(*graph, new_edge_weights, new_edge_ids, new_edge_types, new_number_map) = cugraph::create_graph_from_edgelist( handle_, - std::nullopt, + std::move(vertex_list), std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_weights), @@ -187,7 +237,7 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { std::move(edgelist_edge_types), cugraph::graph_properties_t{properties_->is_symmetric, properties_->is_multigraph}, renumber_, - check_); + do_expensive_check_); if (renumber_) { *number_map = std::move(new_number_map.value()); @@ -204,90 +254,39 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { if (new_edge_types) { *edge_types = std::move(new_edge_types.value()); } // Set up return - auto result = new cugraph::c_api::cugraph_graph_t{ - src_->type_, - edge_type_, - weights_ ? weights_->type_ : cugraph_data_type_id_t::FLOAT32, - edge_type_ids_ ? edge_type_ids_->type_ : cugraph_data_type_id_t::INT32, - store_transposed, - multi_gpu, - graph, - number_map, - new_edge_weights ? edge_weights : nullptr, - new_edge_ids ? edge_ids : nullptr, - new_edge_types ? edge_types : nullptr}; + auto result = new cugraph::c_api::cugraph_graph_t{vertex_type_, + edge_type_, + weight_type_, + edge_type_id_type_, + store_transposed, + multi_gpu, + graph, + number_map, + new_edge_weights ? edge_weights : nullptr, + new_edge_ids ? edge_ids : nullptr, + new_edge_types ? edge_types : nullptr}; result_ = reinterpret_cast(result); } } }; -struct destroy_graph_functor : public cugraph::c_api::abstract_functor { - void* graph_; - void* number_map_; - void* edge_weights_; - void* edge_ids_; - void* edge_types_; - - destroy_graph_functor( - void* graph, void* number_map, void* edge_weights, void* edge_ids, void* edge_types) - : abstract_functor(), - graph_(graph), - number_map_(number_map), - edge_weights_(edge_weights), - edge_ids_(edge_ids), - edge_types_(edge_types) - { - } - - template - void operator()() - { - auto internal_graph_pointer = - reinterpret_cast*>(graph_); - - delete internal_graph_pointer; - - auto internal_number_map_pointer = - reinterpret_cast*>(number_map_); - - delete internal_number_map_pointer; - - auto internal_edge_weight_pointer = reinterpret_cast< - cugraph::edge_property_t, - weight_t>*>(edge_weights_); - if (internal_edge_weight_pointer) { delete internal_edge_weight_pointer; } - - auto internal_edge_id_pointer = reinterpret_cast< - cugraph::edge_property_t, - edge_t>*>(edge_ids_); - if (internal_edge_id_pointer) { delete internal_edge_id_pointer; } - - auto internal_edge_type_pointer = reinterpret_cast< - cugraph::edge_property_t, - edge_type_id_t>*>(edge_types_); - if (internal_edge_type_pointer) { delete internal_edge_type_pointer; } - } -}; - } // namespace -extern "C" cugraph_error_code_t cugraph_mg_graph_create( - const cugraph_resource_handle_t* handle, - const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_view_t* src, - const cugraph_type_erased_device_array_view_t* dst, - const cugraph_type_erased_device_array_view_t* weights, - const cugraph_type_erased_device_array_view_t* edge_ids, - const cugraph_type_erased_device_array_view_t* edge_type_ids, +extern "C" cugraph_error_code_t cugraph_graph_create_mg( + cugraph_resource_handle_t const* handle, + cugraph_graph_properties_t const* properties, + cugraph_type_erased_device_array_view_t const* const* vertices, + cugraph_type_erased_device_array_view_t const* const* src, + cugraph_type_erased_device_array_view_t const* const* dst, + cugraph_type_erased_device_array_view_t const* const* weights, + cugraph_type_erased_device_array_view_t const* const* edge_ids, + cugraph_type_erased_device_array_view_t const* const* edge_type_ids, bool_t store_transposed, - size_t num_edges, - bool_t check, + size_t num_arrays, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, cugraph_graph_t** graph, cugraph_error_t** error) { @@ -298,87 +297,198 @@ extern "C" cugraph_error_code_t cugraph_mg_graph_create( *error = nullptr; auto p_handle = reinterpret_cast(handle); + auto p_vertices = + reinterpret_cast( + vertices); auto p_src = - reinterpret_cast(src); + reinterpret_cast(src); auto p_dst = - reinterpret_cast(dst); + reinterpret_cast(dst); auto p_weights = - reinterpret_cast(weights); + reinterpret_cast( + weights); auto p_edge_ids = - reinterpret_cast(edge_ids); + reinterpret_cast( + edge_ids); auto p_edge_type_ids = - reinterpret_cast(edge_type_ids); + reinterpret_cast( + edge_type_ids); + + size_t local_num_edges{0}; + + // + // Determine the type of vertex, weight, edge_type_id across + // multiple input arrays and acros multiple GPUs. Also compute + // the number of edges so we can determine what type to use for + // edge_t + // + cugraph_data_type_id_t vertex_type{cugraph_data_type_id_t::NTYPES}; + cugraph_data_type_id_t weight_type{cugraph_data_type_id_t::NTYPES}; + + for (size_t i = 0; i < num_arrays; ++i) { + CAPI_EXPECTS(p_src[i]->size_ == p_dst[i]->size_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != dst size.", + *error); + + CAPI_EXPECTS(p_src[i]->type_ == p_dst[i]->type_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != dst type.", + *error); + + CAPI_EXPECTS((p_vertices == nullptr) || (p_src[i]->type_ == p_vertices[i]->type_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != vertices type.", + *error); + + CAPI_EXPECTS((weights == nullptr) || (p_weights[i]->size_ == p_src[i]->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != weights size.", + *error); + + local_num_edges += p_src[i]->size_; + + if (vertex_type == cugraph_data_type_id_t::NTYPES) vertex_type = p_src[i]->type_; + + if (weights != nullptr) { + if (weight_type == cugraph_data_type_id_t::NTYPES) weight_type = p_weights[i]->type_; + } - CAPI_EXPECTS(p_src->size_ == p_dst->size_, - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != dst size.", - *error); - CAPI_EXPECTS(p_src->type_ == p_dst->type_, + CAPI_EXPECTS(p_src[i]->type_ == vertex_type, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: all vertex types must match", + *error); + + CAPI_EXPECTS((weights == nullptr) || (p_weights[i]->type_ == weight_type), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: all weight types must match", + *error); + } + + size_t num_edges = cugraph::host_scalar_allreduce(p_handle->handle_->get_comms(), + local_num_edges, + raft::comms::op_t::SUM, + p_handle->handle_->get_stream()); + + auto vertex_types = cugraph::host_scalar_allgather( + p_handle->handle_->get_comms(), static_cast(vertex_type), p_handle->handle_->get_stream()); + + auto weight_types = cugraph::host_scalar_allgather( + p_handle->handle_->get_comms(), static_cast(weight_type), p_handle->handle_->get_stream()); + + if (vertex_type == cugraph_data_type_id_t::NTYPES) { + // Only true if this GPU had no vertex arrays + vertex_type = static_cast( + *std::min_element(vertex_types.begin(), vertex_types.end())); + } + + if (weight_type == cugraph_data_type_id_t::NTYPES) { + // Only true if this GPU had no weight arrays + weight_type = static_cast( + *std::min_element(weight_types.begin(), weight_types.end())); + } + + CAPI_EXPECTS(std::all_of(vertex_types.begin(), + vertex_types.end(), + [vertex_type](auto t) { return vertex_type == static_cast(t); }), CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src type != dst type.", + "different vertex type used on different GPUs", *error); - CAPI_EXPECTS((weights == nullptr) || (p_weights->size_ == p_src->size_), + CAPI_EXPECTS(std::all_of(weight_types.begin(), + weight_types.end(), + [weight_type](auto t) { return weight_type == static_cast(t); }), CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != weights size.", + "different weight type used on different GPUs", *error); cugraph_data_type_id_t edge_type; - cugraph_data_type_id_t weight_type; if (num_edges < int32_threshold) { - edge_type = p_src->type_; + edge_type = static_cast(vertex_types[0]); } else { edge_type = cugraph_data_type_id_t::INT64; } - if (weights != nullptr) { - weight_type = p_weights->type_; - } else { + if (weight_type == cugraph_data_type_id_t::NTYPES) { weight_type = cugraph_data_type_id_t::FLOAT32; } - CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids->type_ == edge_type), - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: Edge id type must match edge type", - *error); + cugraph_data_type_id_t edge_type_id_type{cugraph_data_type_id_t::NTYPES}; - CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids->size_ == p_src->size_), - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != edge id prop size", - *error); + for (size_t i = 0; i < num_arrays; ++i) { + CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids[i]->type_ == edge_type), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: Edge id type must match edge type", + *error); - CAPI_EXPECTS((edge_type_ids == nullptr) || (p_edge_type_ids->size_ == p_src->size_), - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != edge type prop size", - *error); + CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids[i]->size_ == p_src[i]->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge id prop size", + *error); + + if (edge_type_ids != nullptr) { + CAPI_EXPECTS(p_edge_type_ids[i]->size_ == p_src[i]->size_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge type prop size", + *error); + + if (edge_type_id_type == cugraph_data_type_id_t::NTYPES) + edge_type_id_type = p_edge_type_ids[i]->type_; + + CAPI_EXPECTS(p_edge_type_ids[i]->type_ == edge_type_id_type, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge type prop size", + *error); + } + } + + auto edge_type_id_types = cugraph::host_scalar_allgather(p_handle->handle_->get_comms(), + static_cast(edge_type_id_type), + p_handle->handle_->get_stream()); + + if (edge_type_id_type == cugraph_data_type_id_t::NTYPES) { + // Only true if this GPU had no edge_type_id arrays + edge_type_id_type = static_cast( + *std::min_element(edge_type_id_types.begin(), edge_type_id_types.end())); + } + + CAPI_EXPECTS( + std::all_of(edge_type_id_types.begin(), + edge_type_id_types.end(), + [edge_type_id_type](auto t) { return edge_type_id_type == static_cast(t); }), + CUGRAPH_INVALID_INPUT, + "different edge_type_id type used on different GPUs", + *error); - cugraph_data_type_id_t edge_type_id_type; - if (edge_type_ids == nullptr) { + if (edge_type_id_type == cugraph_data_type_id_t::NTYPES) { edge_type_id_type = cugraph_data_type_id_t::INT32; - } else { - edge_type_id_type = p_edge_type_ids->type_; } + // + // Now we know enough to create the graph + // create_graph_functor functor(*p_handle->handle_, properties, + vertex_type, + edge_type, + weight_type, + edge_type_id_type, + p_vertices, p_src, p_dst, p_weights, p_edge_ids, p_edge_type_ids, + num_arrays, bool_t::TRUE, - check, - edge_type); + drop_self_loops, + drop_multi_edges, + do_expensive_check); try { - cugraph::c_api::vertex_dispatcher(p_src->type_, - edge_type, - weight_type, - edge_type_id_type, - store_transposed, - multi_gpu, - functor); + cugraph::c_api::vertex_dispatcher( + vertex_type, edge_type, weight_type, edge_type_id_type, store_transposed, multi_gpu, functor); if (functor.error_code_ != CUGRAPH_SUCCESS) { *error = reinterpret_cast(functor.error_.release()); @@ -394,25 +504,38 @@ extern "C" cugraph_error_code_t cugraph_mg_graph_create( return CUGRAPH_SUCCESS; } +extern "C" cugraph_error_code_t cugraph_mg_graph_create( + cugraph_resource_handle_t const* handle, + cugraph_graph_properties_t const* properties, + cugraph_type_erased_device_array_view_t const* src, + cugraph_type_erased_device_array_view_t const* dst, + cugraph_type_erased_device_array_view_t const* weights, + cugraph_type_erased_device_array_view_t const* edge_ids, + cugraph_type_erased_device_array_view_t const* edge_type_ids, + bool_t store_transposed, + size_t num_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) +{ + return cugraph_graph_create_mg(handle, + properties, + NULL, + &src, + &dst, + (weights == nullptr) ? nullptr : &weights, + (edge_ids == nullptr) ? nullptr : &edge_ids, + (edge_type_ids == nullptr) ? nullptr : &edge_type_ids, + store_transposed, + 1, + FALSE, + FALSE, + do_expensive_check, + graph, + error); +} + extern "C" void cugraph_mg_graph_free(cugraph_graph_t* ptr_graph) { - if (ptr_graph != NULL) { - auto internal_pointer = reinterpret_cast(ptr_graph); - - destroy_graph_functor functor(internal_pointer->graph_, - internal_pointer->number_map_, - internal_pointer->edge_weights_, - internal_pointer->edge_ids_, - internal_pointer->edge_types_); - - cugraph::c_api::vertex_dispatcher(internal_pointer->vertex_type_, - internal_pointer->edge_type_, - internal_pointer->weight_type_, - internal_pointer->edge_type_id_type_, - internal_pointer->store_transposed_, - internal_pointer->multi_gpu_, - functor); - - delete internal_pointer; - } + if (ptr_graph != NULL) { cugraph_graph_free(ptr_graph); } } diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index 9536869f123..7793458b53a 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -33,35 +33,44 @@ namespace { struct create_graph_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; cugraph_graph_properties_t const* properties_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* vertices_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* src_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids_; bool_t renumber_; + bool_t drop_self_loops_; + bool_t drop_multi_edges_; bool_t do_expensive_check_; cugraph_data_type_id_t edge_type_; cugraph::c_api::cugraph_graph_t* result_{}; create_graph_functor(raft::handle_t const& handle, cugraph_graph_properties_t const* properties, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* vertices, cugraph::c_api::cugraph_type_erased_device_array_view_t const* src, cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst, cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights, cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids, cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids, bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, bool_t do_expensive_check, cugraph_data_type_id_t edge_type) : abstract_functor(), properties_(properties), handle_(handle), + vertices_(vertices), src_(src), dst_(dst), weights_(weights), edge_ids_(edge_ids), edge_type_ids_(edge_type_ids), renumber_(renumber), + drop_self_loops_(drop_self_loops), + drop_multi_edges_(drop_multi_edges), do_expensive_check_(do_expensive_check), edge_type_(edge_type) { @@ -99,6 +108,18 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { edge_type_id_t>> new_edge_types{std::nullopt}; + std::optional> vertex_list = + vertices_ ? std::make_optional( + rmm::device_uvector(vertices_->size_, handle_.get_stream())) + : std::nullopt; + + if (vertex_list) { + raft::copy(vertex_list->data(), + vertices_->as_type(), + vertices_->size_, + handle_.get_stream()); + } + rmm::device_uvector edgelist_srcs(src_->size_, handle_.get_stream()); rmm::device_uvector edgelist_dsts(dst_->size_, handle_.get_stream()); @@ -160,6 +181,28 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { cugraph::graph_view_t, edge_type_id_t>(handle_); + if (drop_self_loops_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_self_loops(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + + if (drop_multi_edges_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_multi_edges(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + std::tie(*graph, new_edge_weights, new_edge_ids, new_edge_types, new_number_map) = cugraph::create_graph_from_edgelist( handle_, - std::nullopt, + std::move(vertex_list), std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_weights), @@ -279,6 +322,12 @@ struct create_graph_csr_functor : public cugraph::c_api::abstract_functor { edge_type_id_t>> new_edge_types{std::nullopt}; + std::optional> vertex_list = std::make_optional( + rmm::device_uvector(offsets_->size_ - 1, handle_.get_stream())); + + cugraph::detail::sequence_fill( + handle_.get_stream(), vertex_list->data(), vertex_list->size(), vertex_t{0}); + rmm::device_uvector edgelist_srcs(0, handle_.get_stream()); rmm::device_uvector edgelist_dsts(indices_->size_, handle_.get_stream()); @@ -354,7 +403,7 @@ struct create_graph_csr_functor : public cugraph::c_api::abstract_functor { store_transposed, multi_gpu>( handle_, - std::nullopt, + std::move(vertex_list), std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_weights), @@ -452,9 +501,10 @@ struct destroy_graph_functor : public cugraph::c_api::abstract_functor { } // namespace -extern "C" cugraph_error_code_t cugraph_sg_graph_create( +extern "C" cugraph_error_code_t cugraph_graph_create_sg( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* vertices, const cugraph_type_erased_device_array_view_t* src, const cugraph_type_erased_device_array_view_t* dst, const cugraph_type_erased_device_array_view_t* weights, @@ -462,6 +512,8 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( const cugraph_type_erased_device_array_view_t* edge_type_ids, bool_t store_transposed, bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, bool_t do_expensive_check, cugraph_graph_t** graph, cugraph_error_t** error) @@ -473,6 +525,8 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( *error = nullptr; auto p_handle = reinterpret_cast(handle); + auto p_vertices = + reinterpret_cast(vertices); auto p_src = reinterpret_cast(src); auto p_dst = @@ -488,6 +542,12 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( CUGRAPH_INVALID_INPUT, "Invalid input arguments: src size != dst size.", *error); + + CAPI_EXPECTS((p_vertices == nullptr) || (p_src->type_ == p_vertices->type_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != vertices type.", + *error); + CAPI_EXPECTS(p_src->type_ == p_dst->type_, CUGRAPH_INVALID_INPUT, "Invalid input arguments: src type != dst type.", @@ -533,12 +593,15 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( ::create_graph_functor functor(*p_handle->handle_, properties, + p_vertices, p_src, p_dst, p_weights, p_edge_ids, p_edge_type_ids, renumber, + drop_self_loops, + drop_multi_edges, do_expensive_check, edge_type); @@ -565,7 +628,38 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( return CUGRAPH_SUCCESS; } -cugraph_error_code_t cugraph_sg_graph_create_from_csr( +extern "C" cugraph_error_code_t cugraph_sg_graph_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) +{ + return cugraph_graph_create_sg(handle, + properties, + NULL, + src, + dst, + weights, + edge_ids, + edge_type_ids, + store_transposed, + renumber, + FALSE, + FALSE, + do_expensive_check, + graph, + error); +} + +cugraph_error_code_t cugraph_graph_create_sg_from_csr( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, const cugraph_type_erased_device_array_view_t* offsets, @@ -662,23 +756,55 @@ cugraph_error_code_t cugraph_sg_graph_create_from_csr( return CUGRAPH_SUCCESS; } -extern "C" void cugraph_sg_graph_free(cugraph_graph_t* ptr_graph) +cugraph_error_code_t cugraph_sg_graph_create_from_csr( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* offsets, + const cugraph_type_erased_device_array_view_t* indices, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) { - auto internal_pointer = reinterpret_cast(ptr_graph); - - destroy_graph_functor functor(internal_pointer->graph_, - internal_pointer->number_map_, - internal_pointer->edge_weights_, - internal_pointer->edge_ids_, - internal_pointer->edge_types_); - - cugraph::c_api::vertex_dispatcher(internal_pointer->vertex_type_, - internal_pointer->edge_type_, - internal_pointer->weight_type_, - internal_pointer->edge_type_id_type_, - internal_pointer->store_transposed_, - internal_pointer->multi_gpu_, - functor); - - delete internal_pointer; + return cugraph_graph_create_sg_from_csr(handle, + properties, + offsets, + indices, + weights, + edge_ids, + edge_type_ids, + store_transposed, + renumber, + do_expensive_check, + graph, + error); } + +extern "C" void cugraph_graph_free(cugraph_graph_t* ptr_graph) +{ + if (ptr_graph != NULL) { + auto internal_pointer = reinterpret_cast(ptr_graph); + + destroy_graph_functor functor(internal_pointer->graph_, + internal_pointer->number_map_, + internal_pointer->edge_weights_, + internal_pointer->edge_ids_, + internal_pointer->edge_types_); + + cugraph::c_api::vertex_dispatcher(internal_pointer->vertex_type_, + internal_pointer->edge_type_, + internal_pointer->weight_type_, + internal_pointer->edge_type_id_type_, + internal_pointer->store_transposed_, + internal_pointer->multi_gpu_, + functor); + + delete internal_pointer; + } +} + +extern "C" void cugraph_sg_graph_free(cugraph_graph_t* ptr_graph) { cugraph_graph_free(ptr_graph); } diff --git a/cpp/src/c_api/induced_subgraph.cpp b/cpp/src/c_api/induced_subgraph.cpp index a1bbcb60825..ac56301e231 100644 --- a/cpp/src/c_api/induced_subgraph.cpp +++ b/cpp/src/c_api/induced_subgraph.cpp @@ -147,11 +147,14 @@ struct induced_subgraph_functor : public cugraph::c_api::abstract_functor { graph_view.vertex_partition_range_lasts(), do_expensive_check_); + // FIXME: Add support for edge_id and edge_type_id. result_ = new cugraph::c_api::cugraph_induced_subgraph_result_t{ new cugraph::c_api::cugraph_type_erased_device_array_t(src, graph_->vertex_type_), new cugraph::c_api::cugraph_type_erased_device_array_t(dst, graph_->vertex_type_), wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*wgt, graph_->weight_type_) : NULL, + NULL, + NULL, new cugraph::c_api::cugraph_type_erased_device_array_t(graph_offsets, SIZE_T)}; } } diff --git a/cpp/src/c_api/induced_subgraph_result.cpp b/cpp/src/c_api/induced_subgraph_result.cpp index b9ad0e0d66f..5226872d404 100644 --- a/cpp/src/c_api/induced_subgraph_result.cpp +++ b/cpp/src/c_api/induced_subgraph_result.cpp @@ -45,6 +45,28 @@ extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get internal_pointer->wgt_->view()); } +extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_result_t* induced_subgraph) +{ + auto internal_pointer = + reinterpret_cast(induced_subgraph); + return (internal_pointer->edge_ids_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->edge_ids_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_result_t* induced_subgraph) +{ + auto internal_pointer = + reinterpret_cast(induced_subgraph); + return (internal_pointer->edge_type_ids_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->edge_type_ids_->view()); +} + extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_subgraph_offsets( cugraph_induced_subgraph_result_t* induced_subgraph) { @@ -62,6 +84,8 @@ extern "C" void cugraph_induced_subgraph_result_free( delete internal_pointer->src_; delete internal_pointer->dst_; delete internal_pointer->wgt_; + delete internal_pointer->edge_ids_; + delete internal_pointer->edge_type_ids_; delete internal_pointer->subgraph_offsets_; delete internal_pointer; } diff --git a/cpp/src/c_api/induced_subgraph_result.hpp b/cpp/src/c_api/induced_subgraph_result.hpp index acc99b617f4..6f02a699605 100644 --- a/cpp/src/c_api/induced_subgraph_result.hpp +++ b/cpp/src/c_api/induced_subgraph_result.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,8 @@ struct cugraph_induced_subgraph_result_t { cugraph_type_erased_device_array_t* src_{}; cugraph_type_erased_device_array_t* dst_{}; cugraph_type_erased_device_array_t* wgt_{}; + cugraph_type_erased_device_array_t* edge_ids_{}; + cugraph_type_erased_device_array_t* edge_type_ids_{}; cugraph_type_erased_device_array_t* subgraph_offsets_{}; }; diff --git a/cpp/src/c_api/legacy_k_truss.cpp b/cpp/src/c_api/legacy_k_truss.cpp index 90e0894783a..90db9fc133c 100644 --- a/cpp/src/c_api/legacy_k_truss.cpp +++ b/cpp/src/c_api/legacy_k_truss.cpp @@ -123,12 +123,15 @@ struct k_truss_functor : public cugraph::c_api::abstract_functor { raft::update_device( edge_offsets.data(), h_edge_offsets.data(), h_edge_offsets.size(), handle_.get_stream()); + // FIXME: Add support for edge_id and edge_type_id. result_ = new cugraph::c_api::cugraph_induced_subgraph_result_t{ new cugraph::c_api::cugraph_type_erased_device_array_t(result_src, graph_->vertex_type_), new cugraph::c_api::cugraph_type_erased_device_array_t(result_dst, graph_->vertex_type_), wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*result_wgt, graph_->weight_type_) : NULL, + NULL, + NULL, new cugraph::c_api::cugraph_type_erased_device_array_t(edge_offsets, cugraph_data_type_id_t::SIZE_T)}; } diff --git a/cpp/src/c_api/resource_handle.cpp b/cpp/src/c_api/resource_handle.cpp index 767a6f0add6..75b9537ef49 100644 --- a/cpp/src/c_api/resource_handle.cpp +++ b/cpp/src/c_api/resource_handle.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,3 +41,10 @@ extern "C" int cugraph_resource_handle_get_rank(const cugraph_resource_handle_t* auto& comm = internal->handle_->get_comms(); return static_cast(comm.get_rank()); } + +extern "C" int cugraph_resource_handle_get_comm_size(const cugraph_resource_handle_t* handle) +{ + auto internal = reinterpret_cast(handle); + auto& comm = internal->handle_->get_comms(); + return static_cast(comm.get_size()); +} diff --git a/cpp/src/centrality/eigenvector_centrality_impl.cuh b/cpp/src/centrality/eigenvector_centrality_impl.cuh index 291abf18455..8d1bea4004d 100644 --- a/cpp/src/centrality/eigenvector_centrality_impl.cuh +++ b/cpp/src/centrality/eigenvector_centrality_impl.cuh @@ -96,7 +96,8 @@ rmm::device_uvector eigenvector_centrality( centralities.end(), old_centralities.data()); - update_edge_src_property(handle, pull_graph_view, centralities.begin(), edge_src_centralities); + update_edge_src_property( + handle, pull_graph_view, old_centralities.begin(), edge_src_centralities); if (edge_weight_view) { per_v_transform_reduce_incoming_e( @@ -122,6 +123,13 @@ rmm::device_uvector eigenvector_centrality( centralities.begin()); } + thrust::transform(handle.get_thrust_policy(), + centralities.begin(), + centralities.end(), + old_centralities.begin(), + centralities.begin(), + thrust::plus()); + // Normalize the centralities auto hypotenuse = sqrt(transform_reduce_v( handle, diff --git a/cpp/src/centrality/katz_centrality_impl.cuh b/cpp/src/centrality/katz_centrality_impl.cuh index 202d00a5771..ac31043d862 100644 --- a/cpp/src/centrality/katz_centrality_impl.cuh +++ b/cpp/src/centrality/katz_centrality_impl.cuh @@ -74,8 +74,6 @@ void katz_centrality( CUGRAPH_EXPECTS(epsilon >= 0.0, "Invalid input argument: epsilon should be non-negative."); if (do_expensive_check) { - // FIXME: should I check for betas? - if (has_initial_guess) { auto num_negative_values = count_if_v(handle, pull_graph_view, katz_centralities, [] __device__(auto, auto val) { diff --git a/cpp/src/community/detail/common_methods.cuh b/cpp/src/community/detail/common_methods.cuh index b388ba53e81..f67d4d939ad 100644 --- a/cpp/src/community/detail/common_methods.cuh +++ b/cpp/src/community/detail/common_methods.cuh @@ -52,7 +52,7 @@ struct is_bitwise_comparable> : std::true_type {}; namespace cugraph { namespace detail { -// a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used +// FIXME: a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used template struct key_aggregated_edge_op_t { weight_t total_edge_weight{}; @@ -80,7 +80,7 @@ struct key_aggregated_edge_op_t { } }; -// a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used +// FIXME: a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used template struct reduce_op_t { using type = thrust::tuple; @@ -100,7 +100,28 @@ struct reduce_op_t { } }; -// a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used +// FIXME: a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used +template +struct count_updown_moves_op_t { + bool up_down{}; + __device__ auto operator()(thrust::tuple> p) const + { + vertex_t old_cluster = thrust::get<0>(p); + auto new_cluster_gain_pair = thrust::get<1>(p); + vertex_t new_cluster = thrust::get<0>(new_cluster_gain_pair); + weight_t delta_modularity = thrust::get<1>(new_cluster_gain_pair); + + auto result_assignment = + (delta_modularity > weight_t{0}) + ? (((new_cluster > old_cluster) != up_down) ? old_cluster : new_cluster) + : old_cluster; + + return (delta_modularity > weight_t{0}) + ? (((new_cluster > old_cluster) != up_down) ? false : true) + : false; + } +}; +// FIXME: a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used template struct cluster_update_op_t { bool up_down{}; @@ -115,7 +136,7 @@ struct cluster_update_op_t { } }; -// a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used +// FIXME: a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used template struct return_edge_weight_t { __device__ auto operator()( @@ -125,7 +146,7 @@ struct return_edge_weight_t { } }; -// a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used +// FIXME: a workaround for cudaErrorInvalidDeviceFunction error when device lambda is used template struct return_one_t { __device__ auto operator()( @@ -394,6 +415,21 @@ rmm::device_uvector update_clustering_by_delta_modularity( detail::reduce_op_t{}, cugraph::get_dataframe_buffer_begin(output_buffer)); + int nr_moves = thrust::count_if( + handle.get_thrust_policy(), + thrust::make_zip_iterator(thrust::make_tuple( + next_clusters_v.begin(), cugraph::get_dataframe_buffer_begin(output_buffer))), + thrust::make_zip_iterator( + thrust::make_tuple(next_clusters_v.end(), cugraph::get_dataframe_buffer_end(output_buffer))), + detail::count_updown_moves_op_t{up_down}); + + if (multi_gpu) { + nr_moves = host_scalar_allreduce( + handle.get_comms(), nr_moves, raft::comms::op_t::SUM, handle.get_stream()); + } + + if (nr_moves == 0) { up_down = !up_down; } + thrust::transform(handle.get_thrust_policy(), next_clusters_v.begin(), next_clusters_v.end(), diff --git a/cpp/src/community/detail/refine_impl.cuh b/cpp/src/community/detail/refine_impl.cuh index e811aafc776..ebaae498d04 100644 --- a/cpp/src/community/detail/refine_impl.cuh +++ b/cpp/src/community/detail/refine_impl.cuh @@ -64,7 +64,7 @@ struct leiden_key_aggregated_edge_op_t { weight_t total_edge_weight{}; weight_t resolution{}; // resolution parameter weight_t theta{}; // scaling factor - raft::random::DeviceState device_state{}; + raft::random::DeviceState& device_state; __device__ auto operator()( vertex_t src, vertex_t neighboring_leiden_cluster, @@ -89,8 +89,9 @@ struct leiden_key_aggregated_edge_op_t { // E(Cr, S-Cr) > ||Cr||*(||S|| -||Cr||) bool is_dst_leiden_cluster_well_connected = - dst_leiden_cut_to_louvain > - resolution * dst_leiden_volume * (louvain_cluster_volume - dst_leiden_volume); + dst_leiden_cut_to_louvain > resolution * dst_leiden_volume * + (louvain_cluster_volume - dst_leiden_volume) / + total_edge_weight; // E(v, Cr-v) - ||v||* ||Cr-v||/||V(G)|| // aggregated_weight_to_neighboring_leiden_cluster == E(v, Cr-v)? @@ -98,11 +99,11 @@ struct leiden_key_aggregated_edge_op_t { weight_t mod_gain = -1.0; if (is_src_active > 0) { if ((louvain_of_dst_leiden_cluster == src_louvain_cluster) && - is_dst_leiden_cluster_well_connected) { + (dst_leiden_cluster_id != src_leiden_cluster) && is_dst_leiden_cluster_well_connected) { mod_gain = aggregated_weight_to_neighboring_leiden_cluster - - resolution * src_weighted_deg * (dst_leiden_volume - src_weighted_deg) / - total_edge_weight; - + resolution * src_weighted_deg * dst_leiden_volume / total_edge_weight; +// FIXME: Disable random moves in refinement phase for now. +#if 0 weight_t random_number{0.0}; if (mod_gain > 0.0) { auto flat_id = uint64_t{threadIdx.x + blockIdx.x * blockDim.x}; @@ -117,6 +118,8 @@ struct leiden_key_aggregated_edge_op_t { ? __expf(static_cast((2.0 * mod_gain) / (theta * total_edge_weight))) * random_number : -1.0; +#endif + mod_gain = mod_gain > 0.0 ? mod_gain : -1.0; } } @@ -240,11 +243,12 @@ refine_clustering( wcut_deg_and_cluster_vol_triple_begin, wcut_deg_and_cluster_vol_triple_end, singleton_and_connected_flags.begin(), - [resolution] __device__(auto wcut_wdeg_and_louvain_volume) { + [resolution, total_edge_weight] __device__(auto wcut_wdeg_and_louvain_volume) { auto wcut = thrust::get<0>(wcut_wdeg_and_louvain_volume); auto wdeg = thrust::get<1>(wcut_wdeg_and_louvain_volume); auto louvain_volume = thrust::get<2>(wcut_wdeg_and_louvain_volume); - return wcut > (resolution * wdeg * (louvain_volume - wdeg)); + return wcut > + (resolution * wdeg * (louvain_volume - wdeg) / total_edge_weight); }); edge_src_property_t src_louvain_cluster_weight_cache(handle); @@ -478,7 +482,7 @@ refine_clustering( auto values_for_leiden_cluster_keys = thrust::make_zip_iterator( thrust::make_tuple(refined_community_volumes.begin(), refined_community_cuts.begin(), - leiden_keys_used_in_edge_reduction.begin(), // redundant + leiden_keys_used_in_edge_reduction.begin(), louvain_of_leiden_keys_used_in_edge_reduction.begin())); using value_t = thrust::tuple; diff --git a/cpp/src/community/flatten_dendrogram.hpp b/cpp/src/community/flatten_dendrogram.hpp index 9a0c103c01f..eac20389765 100644 --- a/cpp/src/community/flatten_dendrogram.hpp +++ b/cpp/src/community/flatten_dendrogram.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,4 +59,31 @@ void partition_at_level(raft::handle_t const& handle, }); } +template +void leiden_partition_at_level(raft::handle_t const& handle, + Dendrogram const& dendrogram, + vertex_t* d_partition, + size_t level) +{ + vertex_t local_num_verts = dendrogram.get_level_size_nocheck(0); + raft::copy( + d_partition, dendrogram.get_level_ptr_nocheck(0), local_num_verts, handle.get_stream()); + + rmm::device_uvector local_vertex_ids_v(local_num_verts, handle.get_stream()); + + std::for_each( + thrust::make_counting_iterator(0), + thrust::make_counting_iterator((level - 1) / 2), + [&handle, &dendrogram, &local_vertex_ids_v, &d_partition, local_num_verts](size_t l) { + cugraph::relabel( + handle, + std::tuple(dendrogram.get_level_ptr_nocheck(2 * l + 1), + dendrogram.get_level_ptr_nocheck(2 * l + 2)), + dendrogram.get_level_size_nocheck(2 * l + 1), + d_partition, + local_num_verts, + false); + }); +} + } // namespace cugraph diff --git a/cpp/src/community/leiden_impl.cuh b/cpp/src/community/leiden_impl.cuh index a9faf2f2d82..b6e20272de9 100644 --- a/cpp/src/community/leiden_impl.cuh +++ b/cpp/src/community/leiden_impl.cuh @@ -43,6 +43,34 @@ void check_clustering(graph_view_t const& gr if (graph_view.local_vertex_partition_range_size() > 0) CUGRAPH_EXPECTS(clustering != nullptr, "Invalid input argument: clustering is null"); } +template +vertex_t remove_duplicates(raft::handle_t const& handle, rmm::device_uvector& input_array) +{ + thrust::sort(handle.get_thrust_policy(), input_array.begin(), input_array.end()); + + auto nr_unique_elements = static_cast(thrust::distance( + input_array.begin(), + thrust::unique(handle.get_thrust_policy(), input_array.begin(), input_array.end()))); + + input_array.resize(nr_unique_elements, handle.get_stream()); + + if constexpr (multi_gpu) { + input_array = cugraph::detail::shuffle_ext_vertices_to_local_gpu_by_vertex_partitioning( + handle, std::move(input_array)); + + thrust::sort(handle.get_thrust_policy(), input_array.begin(), input_array.end()); + + nr_unique_elements = static_cast(thrust::distance( + input_array.begin(), + thrust::unique(handle.get_thrust_policy(), input_array.begin(), input_array.end()))); + + input_array.resize(nr_unique_elements, handle.get_stream()); + + nr_unique_elements = host_scalar_allreduce( + handle.get_comms(), nr_unique_elements, raft::comms::op_t::SUM, handle.get_stream()); + } + return nr_unique_elements; +} template >, weight_t> leiden( rmm::device_uvector louvain_of_refined_graph(0, handle.get_stream()); // #V - while (dendrogram->num_levels() < max_level) { + while (dendrogram->num_levels() < 2 * max_level + 1) { // // Initialize every cluster to reference each vertex to itself // @@ -353,40 +381,8 @@ std::pair>, weight_t> leiden( dendrogram->current_level_begin(), dendrogram->current_level_begin() + dendrogram->current_level_size(), copied_louvain_partition.begin()); - - thrust::sort( - handle.get_thrust_policy(), copied_louvain_partition.begin(), copied_louvain_partition.end()); - auto nr_unique_louvain_clusters = - static_cast(thrust::distance(copied_louvain_partition.begin(), - thrust::unique(handle.get_thrust_policy(), - copied_louvain_partition.begin(), - copied_louvain_partition.end()))); - - copied_louvain_partition.resize(nr_unique_louvain_clusters, handle.get_stream()); - - if constexpr (graph_view_t::is_multi_gpu) { - copied_louvain_partition = - cugraph::detail::shuffle_ext_vertices_to_local_gpu_by_vertex_partitioning( - handle, std::move(copied_louvain_partition)); - - thrust::sort(handle.get_thrust_policy(), - copied_louvain_partition.begin(), - copied_louvain_partition.end()); - - nr_unique_louvain_clusters = - static_cast(thrust::distance(copied_louvain_partition.begin(), - thrust::unique(handle.get_thrust_policy(), - copied_louvain_partition.begin(), - copied_louvain_partition.end()))); - - copied_louvain_partition.resize(nr_unique_louvain_clusters, handle.get_stream()); - - nr_unique_louvain_clusters = host_scalar_allreduce(handle.get_comms(), - nr_unique_louvain_clusters, - raft::comms::op_t::SUM, - handle.get_stream()); - } + remove_duplicates(handle, copied_louvain_partition); terminate = terminate || (nr_unique_louvain_clusters == current_graph_view.number_of_vertices()); @@ -481,6 +477,15 @@ std::pair>, weight_t> leiden( (*cluster_assignment).data(), (*cluster_assignment).size(), false); + // louvain assignment of aggregated graph which is necessary to flatten dendrogram + dendrogram->add_level(current_graph_view.local_vertex_partition_range_first(), + current_graph_view.local_vertex_partition_range_size(), + handle.get_stream()); + + raft::copy(dendrogram->current_level_begin(), + (*cluster_assignment).begin(), + (*cluster_assignment).size(), + handle.get_stream()); louvain_of_refined_graph.resize(current_graph_view.local_vertex_partition_range_size(), handle.get_stream()); @@ -492,47 +497,6 @@ std::pair>, weight_t> leiden( } } - // Relabel dendrogram - vertex_t local_cluster_id_first{0}; - if constexpr (multi_gpu) { - auto unique_cluster_range_lasts = cugraph::partition_manager::compute_partition_range_lasts( - handle, static_cast(copied_louvain_partition.size())); - - auto& comm = handle.get_comms(); - auto const comm_size = comm.get_size(); - auto const comm_rank = comm.get_rank(); - auto& major_comm = handle.get_subcomm(cugraph::partition_manager::major_comm_name()); - auto const major_comm_size = major_comm.get_size(); - auto const major_comm_rank = major_comm.get_rank(); - auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); - auto const minor_comm_size = minor_comm.get_size(); - auto const minor_comm_rank = minor_comm.get_rank(); - - auto vertex_partition_id = - partition_manager::compute_vertex_partition_id_from_graph_subcomm_ranks( - major_comm_size, minor_comm_size, major_comm_rank, minor_comm_rank); - - local_cluster_id_first = vertex_partition_id == 0 - ? vertex_t{0} - : unique_cluster_range_lasts[vertex_partition_id - 1]; - } - - rmm::device_uvector numbering_indices(copied_louvain_partition.size(), - handle.get_stream()); - detail::sequence_fill(handle.get_stream(), - numbering_indices.data(), - numbering_indices.size(), - local_cluster_id_first); - - relabel( - handle, - std::make_tuple(static_cast(copied_louvain_partition.begin()), - static_cast(numbering_indices.begin())), - copied_louvain_partition.size(), - dendrogram->current_level_begin(), - dendrogram->current_level_size(), - false); - copied_louvain_partition.resize(0, handle.get_stream()); copied_louvain_partition.shrink_to_fit(handle.get_stream()); @@ -550,23 +514,71 @@ std::pair>, weight_t> leiden( return std::make_pair(std::move(dendrogram), best_modularity); } -// FIXME: Can we have a common flatten_dendrogram to be used by both -// Louvain and Leiden, and possibly other clustering methods? +template +void relabel_cluster_ids(raft::handle_t const& handle, + rmm::device_uvector& unique_cluster_ids, + vertex_t* clustering, + size_t num_nodes) +{ + vertex_t local_cluster_id_first{0}; + if constexpr (multi_gpu) { + auto unique_cluster_range_lasts = cugraph::partition_manager::compute_partition_range_lasts( + handle, static_cast(unique_cluster_ids.size())); + + auto& comm = handle.get_comms(); + auto const comm_size = comm.get_size(); + auto const comm_rank = comm.get_rank(); + auto& major_comm = handle.get_subcomm(cugraph::partition_manager::major_comm_name()); + auto const major_comm_size = major_comm.get_size(); + auto const major_comm_rank = major_comm.get_rank(); + auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); + auto const minor_comm_size = minor_comm.get_size(); + auto const minor_comm_rank = minor_comm.get_rank(); + + auto vertex_partition_id = + partition_manager::compute_vertex_partition_id_from_graph_subcomm_ranks( + major_comm_size, minor_comm_size, major_comm_rank, minor_comm_rank); + + local_cluster_id_first = + vertex_partition_id == 0 ? vertex_t{0} : unique_cluster_range_lasts[vertex_partition_id - 1]; + } + + rmm::device_uvector numbering_indices(unique_cluster_ids.size(), handle.get_stream()); + detail::sequence_fill(handle.get_stream(), + numbering_indices.data(), + numbering_indices.size(), + local_cluster_id_first); + + relabel( + handle, + std::make_tuple(static_cast(unique_cluster_ids.begin()), + static_cast(numbering_indices.begin())), + unique_cluster_ids.size(), + clustering, + num_nodes, + false); +} + template -void flatten_dendrogram(raft::handle_t const& handle, - graph_view_t const& graph_view, - Dendrogram const& dendrogram, - vertex_t* clustering) +void flatten_leiden_dendrogram(raft::handle_t const& handle, + graph_view_t const& graph_view, + Dendrogram const& dendrogram, + vertex_t* clustering) { - rmm::device_uvector vertex_ids_v(graph_view.number_of_vertices(), handle.get_stream()); + leiden_partition_at_level( + handle, dendrogram, clustering, dendrogram.num_levels()); + + rmm::device_uvector unique_cluster_ids(graph_view.number_of_vertices(), + handle.get_stream()); + thrust::copy(handle.get_thrust_policy(), + clustering, + clustering + graph_view.number_of_vertices(), + unique_cluster_ids.begin()); - thrust::sequence(handle.get_thrust_policy(), - vertex_ids_v.begin(), - vertex_ids_v.end(), - graph_view.local_vertex_partition_range_first()); + remove_duplicates(handle, unique_cluster_ids); - partition_at_level( - handle, dendrogram, vertex_ids_v.data(), clustering, dendrogram.num_levels()); + relabel_cluster_ids( + handle, unique_cluster_ids, clustering, graph_view.number_of_vertices()); } } // namespace detail @@ -588,14 +600,14 @@ std::pair>, weight_t> leiden( } template -void flatten_dendrogram(raft::handle_t const& handle, - graph_view_t const& graph_view, - Dendrogram const& dendrogram, - vertex_t* clustering) +void flatten_leiden_dendrogram(raft::handle_t const& handle, + graph_view_t const& graph_view, + Dendrogram const& dendrogram, + vertex_t* clustering) { CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); - detail::flatten_dendrogram(handle, graph_view, dendrogram, clustering); + detail::flatten_leiden_dendrogram(handle, graph_view, dendrogram, clustering); } template @@ -620,7 +632,7 @@ std::pair leiden( std::tie(dendrogram, modularity) = detail::leiden(handle, rng_state, graph_view, edge_weight_view, max_level, resolution, theta); - detail::flatten_dendrogram(handle, graph_view, *dendrogram, clustering); + detail::flatten_leiden_dendrogram(handle, graph_view, *dendrogram, clustering); return std::make_pair(dendrogram->num_levels(), modularity); } diff --git a/cpp/src/components/weakly_connected_components_impl.cuh b/cpp/src/components/weakly_connected_components_impl.cuh index 615a50ded54..b7b6e139cfa 100644 --- a/cpp/src/components/weakly_connected_components_impl.cuh +++ b/cpp/src/components/weakly_connected_components_impl.cuh @@ -236,18 +236,16 @@ struct v_op_t { auto tag = thrust::get<1>(tagged_v); auto v_offset = vertex_partition.local_vertex_partition_offset_from_vertex_nocheck(thrust::get<0>(tagged_v)); - // FIXME: better switch to atomic_ref after - // https://github.com/nvidia/libcudacxx/milestone/2 - auto old = - atomicCAS(level_components + v_offset, invalid_component_id::value, tag); - if (old != invalid_component_id::value && old != tag) { // conflict + cuda::atomic_ref v_component(*(level_components + v_offset)); + auto old = invalid_component_id::value; + bool success = v_component.compare_exchange_strong(old, tag, cuda::std::memory_order_relaxed); + if (!success && (old != tag)) { // conflict return thrust::make_tuple(thrust::optional{bucket_idx_conflict}, thrust::optional{std::byte{0}} /* dummy */); } else { - auto update = (old == invalid_component_id::value); return thrust::make_tuple( - update ? thrust::optional{bucket_idx_next} : thrust::nullopt, - update ? thrust::optional{std::byte{0}} /* dummy */ : thrust::nullopt); + success ? thrust::optional{bucket_idx_next} : thrust::nullopt, + success ? thrust::optional{std::byte{0}} /* dummy */ : thrust::nullopt); } } @@ -457,33 +455,11 @@ void weakly_connected_components_impl(raft::handle_t const& handle, std::numeric_limits::max()); } - // FIXME: we need to add host_scalar_scatter -#if 1 - rmm::device_uvector d_counts(comm_size, handle.get_stream()); - raft::update_device(d_counts.data(), - init_max_new_root_counts.data(), - init_max_new_root_counts.size(), - handle.get_stream()); - device_bcast( - comm, d_counts.data(), d_counts.data(), d_counts.size(), int{0}, handle.get_stream()); - raft::update_host( - &init_max_new_roots, d_counts.data() + comm_rank, size_t{1}, handle.get_stream()); -#else init_max_new_roots = - host_scalar_scatter(comm, init_max_new_root_counts.data(), int{0}, handle.get_stream()); -#endif + host_scalar_scatter(comm, init_max_new_root_counts, int{0}, handle.get_stream()); } else { - // FIXME: we need to add host_scalar_scatter -#if 1 - rmm::device_uvector d_counts(comm_size, handle.get_stream()); - device_bcast( - comm, d_counts.data(), d_counts.data(), d_counts.size(), int{0}, handle.get_stream()); - raft::update_host( - &init_max_new_roots, d_counts.data() + comm_rank, size_t{1}, handle.get_stream()); -#else init_max_new_roots = - host_scalar_scatter(comm, init_max_new_root_counts.data(), int{0}, handle.get_stream()); -#endif + host_scalar_scatter(comm, std::vector{}, int{0}, handle.get_stream()); } handle.sync_stream(); diff --git a/cpp/src/detail/collect_comm_wrapper.cu b/cpp/src/detail/collect_comm_wrapper.cu new file mode 100644 index 00000000000..7ce2241c677 --- /dev/null +++ b/cpp/src/detail/collect_comm_wrapper.cu @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include +#include + +namespace cugraph { +namespace detail { + +template +rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input) +{ + auto gathered_v = cugraph::device_allgatherv(handle, comm, d_input); + + return gathered_v; +} + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +} // namespace detail +} // namespace cugraph diff --git a/cpp/src/link_analysis/hits_impl.cuh b/cpp/src/link_analysis/hits_impl.cuh index 9badb041218..674046745b1 100644 --- a/cpp/src/link_analysis/hits_impl.cuh +++ b/cpp/src/link_analysis/hits_impl.cuh @@ -112,7 +112,8 @@ std::tuple hits(raft::handle_t const& handle, prev_hubs + graph_view.local_vertex_partition_range_size(), result_t{1.0} / num_vertices); } - for (size_t iter = 0; iter < max_iterations; ++iter) { + size_t iter{0}; + while (true) { // Update current destination authorities property per_v_transform_reduce_incoming_e( handle, @@ -162,17 +163,19 @@ std::tuple hits(raft::handle_t const& handle, thrust::make_zip_iterator(thrust::make_tuple(curr_hubs, prev_hubs)), [] __device__(auto, auto val) { return std::abs(thrust::get<0>(val) - thrust::get<1>(val)); }, result_t{0}); - if (diff_sum < epsilon) { - final_iteration_count = iter; - std::swap(prev_hubs, curr_hubs); - break; - } update_edge_src_property(handle, graph_view, curr_hubs, prev_src_hubs); // Swap pointers for the next iteration // After this swap call, prev_hubs has the latest value of hubs std::swap(prev_hubs, curr_hubs); + iter++; + + if (diff_sum < epsilon) { + break; + } else if (iter >= max_iterations) { + CUGRAPH_FAIL("HITS failed to converge."); + } } if (normalize) { @@ -188,7 +191,7 @@ std::tuple hits(raft::handle_t const& handle, hubs); } - return std::make_tuple(diff_sum, final_iteration_count); + return std::make_tuple(diff_sum, iter); } } // namespace detail diff --git a/cpp/src/mtmg/vertex_result.cu b/cpp/src/mtmg/vertex_result.cu index a669a127f41..97fcd291c87 100644 --- a/cpp/src/mtmg/vertex_result.cu +++ b/cpp/src/mtmg/vertex_result.cu @@ -97,7 +97,7 @@ rmm::device_uvector vertex_result_view_t::gather( return vertex_partition.local_vertex_partition_offset_from_vertex_nocheck(v); }); - thrust::gather(handle.raft_handle().get_thrust_policy(), + thrust::gather(handle.get_thrust_policy(), iter, iter + local_vertices.size(), wrapped.begin(), @@ -118,7 +118,7 @@ rmm::device_uvector vertex_result_view_t::gather( // // Finally, reorder result // - thrust::scatter(handle.raft_handle().get_thrust_policy(), + thrust::scatter(handle.get_thrust_policy(), tmp_result.begin(), tmp_result.end(), vertex_pos.begin(), diff --git a/cpp/src/prims/detail/nbr_intersection.cuh b/cpp/src/prims/detail/nbr_intersection.cuh index 2f30faebb3e..cefc1836fa6 100644 --- a/cpp/src/prims/detail/nbr_intersection.cuh +++ b/cpp/src/prims/detail/nbr_intersection.cuh @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -130,12 +131,14 @@ struct update_rx_major_local_degree_t { int minor_comm_size{}; edge_partition_device_view_t edge_partition{}; + thrust::optional> + edge_partition_e_mask{}; size_t reordered_idx_first{}; size_t local_edge_partition_idx{}; raft::device_span rx_reordered_group_lasts{}; - raft::device_span rx_group_firsts{nullptr}; + raft::device_span rx_group_firsts{}; raft::device_span rx_majors{}; raft::device_span local_degrees_for_rx_majors{}; @@ -151,19 +154,28 @@ struct update_rx_major_local_degree_t { auto major = rx_majors[rx_group_firsts[major_comm_rank * minor_comm_size + local_edge_partition_idx] + offset_in_local_edge_partition]; - edge_t local_degree{}; + vertex_t major_idx{0}; + edge_t local_degree{0}; if (multi_gpu && (edge_partition.major_hypersparse_first() && (major >= *(edge_partition.major_hypersparse_first())))) { auto major_hypersparse_idx = edge_partition.major_hypersparse_idx_from_major_nocheck(major); - local_degree = major_hypersparse_idx - ? edge_partition.local_degree((*(edge_partition.major_hypersparse_first()) - - edge_partition.major_range_first()) + - *major_hypersparse_idx) - : edge_t{0}; + if (major_hypersparse_idx) { + major_idx = + (*(edge_partition.major_hypersparse_first()) - edge_partition.major_range_first()) + + *major_hypersparse_idx; + local_degree = edge_partition.local_degree(major_idx); + } } else { - local_degree = - edge_partition.local_degree(edge_partition.major_offset_from_major_nocheck(major)); + major_idx = edge_partition.major_offset_from_major_nocheck(major); + local_degree = edge_partition.local_degree(major_idx); } + + if (edge_partition_e_mask && (local_degree > edge_t{0})) { + auto local_offset = edge_partition.local_offset(major_idx); + local_degree = static_cast( + count_set_bits((*edge_partition_e_mask).value_first(), local_offset, local_degree)); + } + local_degrees_for_rx_majors[rx_group_firsts[major_comm_rank * minor_comm_size + local_edge_partition_idx] + offset_in_local_edge_partition] = local_degree; @@ -173,7 +185,7 @@ struct update_rx_major_local_degree_t { template struct update_rx_major_local_nbrs_t { int major_comm_size{}; @@ -181,21 +193,24 @@ struct update_rx_major_local_nbrs_t { edge_partition_device_view_t edge_partition{}; edge_partition_e_input_device_view_t edge_partition_e_value_input{}; + thrust::optional> + edge_partition_e_mask{}; size_t reordered_idx_first{}; size_t local_edge_partition_idx{}; raft::device_span rx_reordered_group_lasts{}; - raft::device_span rx_group_firsts{nullptr}; + raft::device_span rx_group_firsts{}; raft::device_span rx_majors{}; raft::device_span local_nbr_offsets_for_rx_majors{}; raft::device_span local_nbrs_for_rx_majors{}; - optional_property_buffer_view_t local_nbrs_properties_for_rx_majors{}; + optional_property_buffer_mutable_view_t local_e_property_values_for_rx_majors{}; __device__ void operator()(size_t idx) { using edge_property_value_t = typename edge_partition_e_input_device_view_t::value_type; - auto it = thrust::upper_bound( + + auto it = thrust::upper_bound( thrust::seq, rx_reordered_group_lasts.begin(), rx_reordered_group_lasts.end(), idx); auto major_comm_rank = static_cast(thrust::distance(rx_reordered_group_lasts.begin(), it)); auto offset_in_local_edge_partition = @@ -204,39 +219,76 @@ struct update_rx_major_local_nbrs_t { auto major = rx_majors[rx_group_firsts[major_comm_rank * minor_comm_size + local_edge_partition_idx] + offset_in_local_edge_partition]; - vertex_t const* indices{nullptr}; - [[maybe_unused]] edge_t edge_offset{0}; + + edge_t edge_offset{0}; edge_t local_degree{0}; if (multi_gpu && (edge_partition.major_hypersparse_first() && (major >= *(edge_partition.major_hypersparse_first())))) { auto major_hypersparse_idx = edge_partition.major_hypersparse_idx_from_major_nocheck(major); if (major_hypersparse_idx) { - thrust::tie(indices, edge_offset, local_degree) = edge_partition.local_edges( + auto major_idx = (*(edge_partition.major_hypersparse_first()) - edge_partition.major_range_first()) + - *major_hypersparse_idx); + *major_hypersparse_idx; + edge_offset = edge_partition.local_offset(major_idx); + local_degree = edge_partition.local_degree(major_idx); } } else { - thrust::tie(indices, edge_offset, local_degree) = - edge_partition.local_edges(edge_partition.major_offset_from_major_nocheck(major)); + auto major_idx = edge_partition.major_offset_from_major_nocheck(major); + edge_offset = edge_partition.local_offset(major_idx); + local_degree = edge_partition.local_degree(major_idx); } - // FIXME: this can lead to thread-divergence with a mix of high-degree and low-degree - // vertices in a single warp (better optimize if this becomes a performance - // bottleneck) - size_t start_offset = + auto indices = edge_partition.indices(); + size_t output_start_offset = local_nbr_offsets_for_rx_majors[rx_group_firsts[major_comm_rank * minor_comm_size + local_edge_partition_idx] + offset_in_local_edge_partition]; - thrust::copy(thrust::seq, - indices, - indices + local_degree, - local_nbrs_for_rx_majors.begin() + start_offset); - if constexpr (!std::is_same_v) { - thrust::copy(thrust::seq, - edge_partition_e_value_input.value_first() + edge_offset, - edge_partition_e_value_input.value_first() + (edge_offset + local_degree), - local_nbrs_properties_for_rx_majors.begin() + start_offset); + // FIXME: this can lead to thread-divergence with a mix of high-degree and low-degree + // vertices in a single warp (better optimize if this becomes a performance + // bottleneck) + + static_assert(!edge_partition_e_input_device_view_t::has_packed_bool_element, "unimplemented."); + if (local_degree > 0) { + if (edge_partition_e_mask) { + auto mask_first = (*edge_partition_e_mask).value_first(); + if constexpr (!std::is_same_v) { + auto input_first = + thrust::make_zip_iterator(indices, edge_partition_e_value_input.value_first()) + + edge_offset; + copy_if_mask_set(input_first, + mask_first, + thrust::make_zip_iterator(local_nbrs_for_rx_majors.begin(), + local_e_property_values_for_rx_majors), + edge_offset, + output_start_offset, + local_degree); + } else { + copy_if_mask_set(indices, + mask_first, + local_nbrs_for_rx_majors.begin(), + edge_offset, + output_start_offset, + local_degree); + } + } else { + if constexpr (!std::is_same_v) { + auto input_first = + thrust::make_zip_iterator(indices, edge_partition_e_value_input.value_first()) + + edge_offset; + thrust::copy(thrust::seq, + input_first, + input_first + local_degree, + thrust::make_zip_iterator(local_nbrs_for_rx_majors.begin(), + local_e_property_values_for_rx_majors) + + output_start_offset); + } else { + thrust::copy(thrust::seq, + indices + edge_offset, + indices + (edge_offset + local_degree), + local_nbrs_for_rx_majors.begin() + output_start_offset); + } + } } } }; @@ -259,36 +311,45 @@ template struct pick_min_degree_t { FirstElementToIdxMap first_element_to_idx_map{}; - raft::device_span first_element_offsets{nullptr}; + raft::device_span first_element_offsets{}; SecondElementToIdxMap second_element_to_idx_map{}; - raft::device_span second_element_offsets{nullptr}; + raft::device_span second_element_offsets{}; edge_partition_device_view_t edge_partition{}; + thrust::optional> + edge_partition_e_mask{}; __device__ edge_t operator()(thrust::tuple pair) const { edge_t local_degree0{0}; vertex_t major0 = thrust::get<0>(pair); if constexpr (std::is_same_v) { + vertex_t major_idx{0}; if constexpr (multi_gpu) { if (edge_partition.major_hypersparse_first() && (major0 >= *(edge_partition.major_hypersparse_first()))) { auto major_hypersparse_idx = edge_partition.major_hypersparse_idx_from_major_nocheck(major0); - local_degree0 = - major_hypersparse_idx - ? edge_partition.local_degree((*(edge_partition.major_hypersparse_first()) - - edge_partition.major_range_first()) + - *major_hypersparse_idx) - : edge_t{0}; + if (major_hypersparse_idx) { + major_idx = + (*(edge_partition.major_hypersparse_first()) - edge_partition.major_range_first()) + + *major_hypersparse_idx; + local_degree0 = edge_partition.local_degree(major_idx); + } } else { - local_degree0 = - edge_partition.local_degree(edge_partition.major_offset_from_major_nocheck(major0)); + major_idx = edge_partition.major_offset_from_major_nocheck(major0); + local_degree0 = edge_partition.local_degree(major_idx); } } else { + major_idx = edge_partition.major_offset_from_major_nocheck(major0); + local_degree0 = edge_partition.local_degree(major_idx); + } + + if (edge_partition_e_mask && (local_degree0 > edge_t{0})) { + auto local_offset = edge_partition.local_offset(major_idx); local_degree0 = - edge_partition.local_degree(edge_partition.major_offset_from_major_nocheck(major0)); + count_set_bits((*edge_partition_e_mask).value_first(), local_offset, local_degree0); } } else { auto idx = first_element_to_idx_map.find(major0); @@ -299,24 +360,31 @@ struct pick_min_degree_t { edge_t local_degree1{0}; vertex_t major1 = thrust::get<1>(pair); if constexpr (std::is_same_v) { + vertex_t major_idx{0}; if constexpr (multi_gpu) { if (edge_partition.major_hypersparse_first() && (major1 >= *(edge_partition.major_hypersparse_first()))) { auto major_hypersparse_idx = edge_partition.major_hypersparse_idx_from_major_nocheck(major1); - local_degree1 = - major_hypersparse_idx - ? edge_partition.local_degree((*(edge_partition.major_hypersparse_first()) - - edge_partition.major_range_first()) + - *major_hypersparse_idx) - : edge_t{0}; + if (major_hypersparse_idx) { + major_idx = + (*(edge_partition.major_hypersparse_first()) - edge_partition.major_range_first()) + + *major_hypersparse_idx; + local_degree1 = edge_partition.local_degree(major_idx); + } } else { - local_degree1 = - edge_partition.local_degree(edge_partition.major_offset_from_major_nocheck(major1)); + major_idx = edge_partition.major_offset_from_major_nocheck(major1); + local_degree1 = edge_partition.local_degree(major_idx); } } else { + major_idx = edge_partition.major_offset_from_major_nocheck(major1); + local_degree1 = edge_partition.local_degree(major_idx); + } + + if (edge_partition_e_mask && (local_degree1 > edge_t{0})) { + auto local_offset = edge_partition.local_offset(major_idx); local_degree1 = - edge_partition.local_degree(edge_partition.major_offset_from_major_nocheck(major1)); + count_set_bits((*edge_partition_e_mask).value_first(), local_offset, local_degree1); } } else { auto idx = second_element_to_idx_map.find(major1); @@ -328,6 +396,71 @@ struct pick_min_degree_t { } }; +template +__device__ edge_t set_intersection_by_key_with_mask(InputKeyIterator0 input_key_first0, + InputKeyIterator1 input_key_first1, + InputValueIterator0 input_value_first0, + InputValueIterator1 input_value_first1, + MaskIterator mask_first, + OutputKeyIterator output_key_first, + OutputValueIterator0 output_value_first0, + OutputValueIterator1 output_value_first1, + edge_t input_start_offset0, + edge_t input_size0, + bool apply_mask0, + edge_t input_start_offset1, + edge_t input_size1, + bool apply_mask1, + size_t output_start_offset) +{ + static_assert( + std::is_same_v::value_type, uint32_t>); + static_assert(std::is_same_v == + std::is_same_v); + + check_bit_set_t check_bit_set{mask_first, edge_t{0}}; + + auto idx0 = input_start_offset0; + auto idx1 = input_start_offset1; + auto output_idx = output_start_offset; + while ((idx0 < (input_start_offset0 + input_size0)) && + (idx1 < (input_start_offset1 + input_size1))) { + bool valid0 = apply_mask0 ? check_bit_set(idx0) : true; + bool valid1 = apply_mask1 ? check_bit_set(idx1) : true; + if (!valid0) { ++idx0; } + if (!valid1) { ++idx1; } + + if (valid0 && valid1) { + auto key0 = *(input_key_first0 + idx0); + auto key1 = *(input_key_first1 + idx1); + if (key0 < key1) { + ++idx0; + } else if (key0 > key1) { + ++idx1; + } else { + *(output_key_first + output_idx) = key0; + if constexpr (!std::is_same_v) { + *(output_value_first0 + output_idx) = *(input_value_first0 + idx0); + *(output_value_first1 + output_idx) = *(input_value_first1 + idx1); + } + ++idx0; + ++idx1; + ++output_idx; + } + } + } + + return (output_idx - output_start_offset); +} + template struct copy_intersecting_nbrs_and_update_intersection_size_t { FirstElementToIdxMap first_element_to_idx_map{}; - raft::device_span first_element_offsets{}; - raft::device_span first_element_indices{nullptr}; - optional_property_buffer_view_t first_element_properties{}; + raft::device_span first_element_offsets{}; + raft::device_span first_element_indices{}; + optional_property_buffer_view_t first_element_edge_property_values{}; SecondElementToIdxMap second_element_to_idx_map{}; - raft::device_span second_element_offsets{}; - raft::device_span second_element_indices{nullptr}; - optional_property_buffer_view_t second_element_properties{}; + raft::device_span second_element_offsets{}; + raft::device_span second_element_indices{}; + optional_property_buffer_view_t second_element_edge_property_values{}; edge_partition_device_view_t edge_partition{}; edge_partition_e_input_device_view_t edge_partition_e_value_input{}; + thrust::optional> + edge_partition_e_mask{}; VertexPairIterator vertex_pair_first; - raft::device_span nbr_intersection_offsets{nullptr}; - raft::device_span nbr_intersection_indices{nullptr}; + raft::device_span nbr_intersection_offsets{}; + raft::device_span nbr_intersection_indices{}; - optional_property_buffer_view_t nbr_intersection_properties0{}; - optional_property_buffer_view_t nbr_intersection_properties1{}; + optional_property_buffer_mutable_view_t nbr_intersection_e_property_values0{}; + optional_property_buffer_mutable_view_t nbr_intersection_e_property_values1{}; vertex_t invalid_id{}; + __device__ edge_t operator()(size_t i) { using edge_property_value_t = typename edge_partition_e_input_device_view_t::value_type; - using optional_const_property_buffer_view_t = - std::conditional_t, - raft::device_span, - std::byte /* dummy */>; auto pair = *(vertex_pair_first + i); - vertex_t const* indices0{nullptr}; - optional_const_property_buffer_view_t properties0{}; + vertex_t const* indices0{}; + std::conditional_t, + edge_property_value_t const*, + void*> + edge_property_values0{}; edge_t local_edge_offset0{0}; edge_t local_degree0{0}; if constexpr (std::is_same_v) { + indices0 = edge_partition.indices(); + if constexpr (!std::is_same_v) { + edge_property_values0 = edge_partition_e_value_input.value_first(); + } + vertex_t major = thrust::get<0>(pair); if constexpr (multi_gpu) { if (edge_partition.major_hypersparse_first() && @@ -379,42 +520,47 @@ struct copy_intersecting_nbrs_and_update_intersection_size_t { auto major_hypersparse_idx = edge_partition.major_hypersparse_idx_from_major_nocheck(major); if (major_hypersparse_idx) { - thrust::tie(indices0, local_edge_offset0, local_degree0) = edge_partition.local_edges( + auto major_idx = (*(edge_partition.major_hypersparse_first()) - edge_partition.major_range_first()) + - *major_hypersparse_idx); + *major_hypersparse_idx; + local_edge_offset0 = edge_partition.local_offset(major_idx); + local_degree0 = edge_partition.local_degree(major_idx); } } else { - thrust::tie(indices0, local_edge_offset0, local_degree0) = - edge_partition.local_edges(edge_partition.major_offset_from_major_nocheck(major)); + auto major_idx = edge_partition.major_offset_from_major_nocheck(major); + local_edge_offset0 = edge_partition.local_offset(major_idx); + local_degree0 = edge_partition.local_degree(major_idx); } } else { - thrust::tie(indices0, local_edge_offset0, local_degree0) = - edge_partition.local_edges(edge_partition.major_offset_from_major_nocheck(major)); + auto major_idx = edge_partition.major_offset_from_major_nocheck(major); + local_edge_offset0 = edge_partition.local_offset(major_idx); + local_degree0 = edge_partition.local_degree(major_idx); } - + } else { + indices0 = first_element_indices.begin(); if constexpr (!std::is_same_v) { - properties0 = raft::device_span( - edge_partition_e_value_input.value_first() + local_edge_offset0, local_degree0); + edge_property_values0 = first_element_edge_property_values; } - } else { auto idx = first_element_to_idx_map.find(thrust::get<0>(pair)); local_edge_offset0 = first_element_offsets[idx]; local_degree0 = static_cast(first_element_offsets[idx + 1] - local_edge_offset0); - indices0 = first_element_indices.begin() + local_edge_offset0; - - if constexpr (!std::is_same_v) { - properties0 = raft::device_span( - first_element_properties.begin() + local_edge_offset0, local_degree0); - } } - vertex_t const* indices1{nullptr}; - optional_const_property_buffer_view_t properties1{}; + vertex_t const* indices1{}; + std::conditional_t, + edge_property_value_t const*, + void*> + edge_property_values1{}; - [[maybe_unused]] edge_t local_edge_offset1{0}; + edge_t local_edge_offset1{0}; edge_t local_degree1{0}; if constexpr (std::is_same_v) { + indices1 = edge_partition.indices(); + if constexpr (!std::is_same_v) { + edge_property_values1 = edge_partition_e_value_input.value_first(); + } + vertex_t major = thrust::get<1>(pair); if constexpr (multi_gpu) { if (edge_partition.major_hypersparse_first() && @@ -422,83 +568,63 @@ struct copy_intersecting_nbrs_and_update_intersection_size_t { auto major_hypersparse_idx = edge_partition.major_hypersparse_idx_from_major_nocheck(major); if (major_hypersparse_idx) { - thrust::tie(indices1, local_edge_offset1, local_degree1) = edge_partition.local_edges( + auto major_idx = (*(edge_partition.major_hypersparse_first()) - edge_partition.major_range_first()) + - *major_hypersparse_idx); + *major_hypersparse_idx; + local_edge_offset1 = edge_partition.local_offset(major_idx); + local_degree1 = edge_partition.local_degree(major_idx); } } else { - thrust::tie(indices1, local_edge_offset1, local_degree1) = - edge_partition.local_edges(edge_partition.major_offset_from_major_nocheck(major)); + auto major_idx = edge_partition.major_offset_from_major_nocheck(major); + local_edge_offset1 = edge_partition.local_offset(major_idx); + local_degree1 = edge_partition.local_degree(major_idx); } } else { - thrust::tie(indices1, local_edge_offset1, local_degree1) = - edge_partition.local_edges(edge_partition.major_offset_from_major_nocheck(major)); + auto major_idx = edge_partition.major_offset_from_major_nocheck(major); + local_edge_offset1 = edge_partition.local_offset(major_idx); + local_degree1 = edge_partition.local_degree(major_idx); } - + } else { + indices1 = second_element_indices.begin(); if constexpr (!std::is_same_v) { - properties1 = raft::device_span( - edge_partition_e_value_input.value_first() + local_edge_offset1, local_degree1); + edge_property_values1 = second_element_edge_property_values; } - } else { auto idx = second_element_to_idx_map.find(thrust::get<1>(pair)); local_edge_offset1 = second_element_offsets[idx]; local_degree1 = static_cast(second_element_offsets[idx + 1] - local_edge_offset1); - indices1 = second_element_indices.begin() + local_edge_offset1; - - if constexpr (!std::is_same_v) { - properties1 = raft::device_span( - second_element_properties.begin() + local_edge_offset1, local_degree1); - } } // FIXME: this can lead to thread-divergence with a mix of high-degree and low-degree // vertices in a single warp (better optimize if this becomes a performance // bottleneck) - auto nbr_intersection_first = nbr_intersection_indices.begin() + nbr_intersection_offsets[i]; - - auto nbr_intersection_last = thrust::set_intersection(thrust::seq, - indices0, - indices0 + local_degree0, - indices1, - indices1 + local_degree1, - nbr_intersection_first); - thrust::fill(thrust::seq, - nbr_intersection_last, - nbr_intersection_indices.begin() + nbr_intersection_offsets[i + 1], - invalid_id); - - auto insection_size = - static_cast(thrust::distance(nbr_intersection_first, nbr_intersection_last)); - if constexpr (!std::is_same_v) { - auto ip0_start = nbr_intersection_properties0.begin() + nbr_intersection_offsets[i]; - - // copy edge properties from first vertex to common neighbors - thrust::transform(thrust::seq, - nbr_intersection_first, - nbr_intersection_last, - ip0_start, - [indices0, local_degree0, properties0] __device__(auto v) { - auto position = - thrust::lower_bound(thrust::seq, indices0, indices0 + local_degree0, v); - return properties0[thrust::distance(indices0, position)]; - }); - - auto ip1_start = nbr_intersection_properties1.begin() + nbr_intersection_offsets[i]; - - // copy edge properties from second vertex to common neighbors - thrust::transform(thrust::seq, - nbr_intersection_first, - nbr_intersection_last, - ip1_start, - [indices1, local_degree1, properties1] __device__(auto v) { - auto position = - thrust::lower_bound(thrust::seq, indices1, indices1 + local_degree1, v); - return properties1[thrust::distance(indices1, position)]; - }); - } - return insection_size; + auto mask_first = edge_partition_e_mask ? (*edge_partition_e_mask).value_first() + : static_cast(nullptr); + auto intersection_size = set_intersection_by_key_with_mask( + indices0, + indices1, + edge_property_values0, + edge_property_values1, + mask_first, + nbr_intersection_indices.begin(), + nbr_intersection_e_property_values0, + nbr_intersection_e_property_values1, + local_edge_offset0, + local_degree0, + (std::is_same_v && edge_partition_e_mask), + local_edge_offset1, + local_degree1, + (std::is_same_v && edge_partition_e_mask), + nbr_intersection_offsets[i]); + + thrust::fill( + thrust::seq, + nbr_intersection_indices.begin() + (nbr_intersection_offsets[i] + intersection_size), + nbr_intersection_indices.begin() + nbr_intersection_offsets[i + 1], + invalid_id); + + return intersection_size; } }; @@ -520,7 +646,8 @@ struct strided_accumulate_t { template + typename optional_property_buffer_view_t, + typename optional_property_buffer_mutable_view_t> struct gatherv_indices_t { size_t output_size{}; int minor_comm_size{}; @@ -530,10 +657,10 @@ struct gatherv_indices_t { raft::device_span combined_nbr_intersection_offsets{}; raft::device_span combined_nbr_intersection_indices{}; - optional_property_buffer_view_t gathered_nbr_intersection_properties0{}; - optional_property_buffer_view_t gathered_nbr_intersection_properties1{}; - optional_property_buffer_view_t combined_nbr_intersection_properties0{}; - optional_property_buffer_view_t combined_nbr_intersection_properties1{}; + optional_property_buffer_view_t gathered_nbr_intersection_e_property_values0{}; + optional_property_buffer_view_t gathered_nbr_intersection_e_property_values1{}; + optional_property_buffer_mutable_view_t combined_nbr_intersection_e_property_values0{}; + optional_property_buffer_mutable_view_t combined_nbr_intersection_e_property_values1{}; __device__ void operator()(size_t i) const { @@ -546,13 +673,13 @@ struct gatherv_indices_t { if constexpr (!std::is_same_v) { auto zipped_gathered_begin = thrust::make_zip_iterator( thrust::make_tuple(gathered_intersection_indices.begin(), - gathered_nbr_intersection_properties0.begin(), - gathered_nbr_intersection_properties1.begin())); + gathered_nbr_intersection_e_property_values0, + gathered_nbr_intersection_e_property_values1)); auto zipped_combined_begin = thrust::make_zip_iterator( thrust::make_tuple(combined_nbr_intersection_indices.begin(), - combined_nbr_intersection_properties0.begin(), - combined_nbr_intersection_properties1.begin())); + combined_nbr_intersection_e_property_values0, + combined_nbr_intersection_e_property_values1)); thrust::copy(thrust::seq, zipped_gathered_begin + gathered_intersection_offsets[output_size * j + i], @@ -694,13 +821,12 @@ nbr_intersection(raft::handle_t const& handle, using optional_property_buffer_view_t = std::conditional_t, - raft::device_span, - std::byte /* dummy */>; - - using optional_nbr_intersected_edge_partitions_t = + edge_property_value_t const*, + void*>; + using optional_property_buffer_mutable_view_t = std::conditional_t, - std::vector>, - std::byte /* dummy */>; + edge_property_value_t*, + void*>; static_assert(std::is_same_v::value_type, thrust::tuple>); @@ -729,19 +855,20 @@ nbr_intersection(raft::handle_t const& handle, "Invalid input arguments: there are invalid input vertex pairs."); } - // 2. Collect neighbor lists for unique second pair elements (for the neighbors within the minor - // range for this GPU); Note that no need to collect for first pair elements as they already - // locally reside. + // 2. Collect neighbor lists (within the minor range for this GPU in multi-GPU) for unique second + // pair elements (all-gathered over minor_comm in multi-GPU); Note that no need to collect for + // first pair elements as they already locally reside. + + auto edge_mask_view = graph_view.edge_mask_view(); std::optional>> major_to_idx_map_ptr{ std::nullopt}; - std::optional> major_nbr_offsets{std::nullopt}; + std::optional> major_nbr_offsets{std::nullopt}; std::optional> major_nbr_indices{std::nullopt}; - [[maybe_unused]] auto major_nbr_properties = + [[maybe_unused]] auto major_e_property_values = cugraph::detail::allocate_optional_dataframe_buffer( 0, handle.get_stream()); - optional_property_buffer_view_t optional_major_nbr_properties{}; if constexpr (GraphViewType::is_multi_gpu) { if (intersect_minor_nbr[1]) { @@ -805,7 +932,7 @@ nbr_intersection(raft::handle_t const& handle, } } - // 2.2 Send majors and group (major_comm_rank, edge_partition_idx) counts + // 2.2 Send majors and group (major_comm_rank, local edge_partition_idx) counts rmm::device_uvector rx_majors(0, handle.get_stream()); std::vector rx_major_counts{}; @@ -859,7 +986,7 @@ nbr_intersection(raft::handle_t const& handle, rmm::device_uvector local_degrees_for_rx_majors(size_t{0}, handle.get_stream()); rmm::device_uvector local_nbrs_for_rx_majors(size_t{0}, handle.get_stream()); - [[maybe_unused]] auto local_nbrs_properties_for_rx_majors = + [[maybe_unused]] auto local_e_property_values_for_rx_majors = cugraph::detail::allocate_optional_dataframe_buffer( 0, handle.get_stream()); @@ -901,6 +1028,13 @@ nbr_intersection(raft::handle_t const& handle, auto edge_partition = edge_partition_device_view_t( graph_view.local_edge_partition_view(i)); + auto edge_partition_e_mask = + edge_mask_view + ? thrust::make_optional< + detail:: + edge_partition_edge_property_device_view_t>( + *edge_mask_view, i) + : thrust::nullopt; auto segment_offsets = graph_view.local_edge_partition_segment_offsets(i); auto reordered_idx_first = (i == size_t{0}) ? size_t{0} : h_rx_reordered_group_lasts[i * major_comm_size - 1]; @@ -913,6 +1047,7 @@ nbr_intersection(raft::handle_t const& handle, major_comm_size, minor_comm_size, edge_partition, + edge_partition_e_mask, reordered_idx_first, i, raft::device_span( @@ -936,22 +1071,28 @@ nbr_intersection(raft::handle_t const& handle, local_nbrs_for_rx_majors.resize( local_nbr_offsets_for_rx_majors.back_element(handle.get_stream()), handle.get_stream()); - optional_property_buffer_view_t optional_local_nbrs_properties{}; + optional_property_buffer_mutable_view_t optional_local_e_property_values{}; if constexpr (!std::is_same_v) { - local_nbrs_properties_for_rx_majors.resize(local_nbrs_for_rx_majors.size(), - handle.get_stream()); - optional_local_nbrs_properties = raft::device_span( - local_nbrs_properties_for_rx_majors.data(), local_nbrs_properties_for_rx_majors.size()); + local_e_property_values_for_rx_majors.resize(local_nbrs_for_rx_majors.size(), + handle.get_stream()); + optional_local_e_property_values = local_e_property_values_for_rx_majors.data(); } for (size_t i = 0; i < graph_view.number_of_local_edge_partitions(); ++i) { auto edge_partition = edge_partition_device_view_t( graph_view.local_edge_partition_view(i)); - auto edge_partition_e_value_input = edge_partition_e_input_device_view_t(edge_value_input, i); + auto edge_partition_e_mask = + edge_mask_view + ? thrust::make_optional< + detail:: + edge_partition_edge_property_device_view_t>( + *edge_mask_view, i) + : thrust::nullopt; + auto segment_offsets = graph_view.local_edge_partition_segment_offsets(i); auto reordered_idx_first = (i == size_t{0}) ? size_t{0} : h_rx_reordered_group_lasts[i * major_comm_size - 1]; @@ -964,12 +1105,13 @@ nbr_intersection(raft::handle_t const& handle, update_rx_major_local_nbrs_t{ major_comm_size, minor_comm_size, edge_partition, edge_partition_e_value_input, + edge_partition_e_mask, reordered_idx_first, i, raft::device_span( @@ -980,7 +1122,7 @@ nbr_intersection(raft::handle_t const& handle, local_nbr_offsets_for_rx_majors.size()), raft::device_span(local_nbrs_for_rx_majors.data(), local_nbrs_for_rx_majors.size()), - optional_local_nbrs_properties}); + optional_local_e_property_values}); } std::vector h_rx_offsets(rx_major_counts.size() + size_t{1}, size_t{0}); @@ -1012,7 +1154,7 @@ nbr_intersection(raft::handle_t const& handle, rmm::device_uvector local_degrees_for_unique_majors(size_t{0}, handle.get_stream()); std::tie(local_degrees_for_unique_majors, std::ignore) = shuffle_values( major_comm, local_degrees_for_rx_majors.begin(), rx_major_counts, handle.get_stream()); - major_nbr_offsets = rmm::device_uvector(local_degrees_for_unique_majors.size() + 1, + major_nbr_offsets = rmm::device_uvector(local_degrees_for_unique_majors.size() + 1, handle.get_stream()); (*major_nbr_offsets).set_element_to_zero_async(size_t{0}, handle.get_stream()); auto degree_first = thrust::make_transform_iterator(local_degrees_for_unique_majors.begin(), @@ -1027,14 +1169,11 @@ nbr_intersection(raft::handle_t const& handle, major_comm, local_nbrs_for_rx_majors.begin(), local_nbr_counts, handle.get_stream()); if constexpr (!std::is_same_v) { - std::tie(major_nbr_properties, std::ignore) = + std::tie(major_e_property_values, std::ignore) = shuffle_values(major_comm, - local_nbrs_properties_for_rx_majors.begin(), + local_e_property_values_for_rx_majors.begin(), local_nbr_counts, handle.get_stream()); - - optional_major_nbr_properties = raft::device_span( - major_nbr_properties.data(), major_nbr_properties.size()); } major_to_idx_map_ptr = std::make_unique>( @@ -1065,11 +1204,11 @@ nbr_intersection(raft::handle_t const& handle, rmm::device_uvector nbr_intersection_offsets(size_t{0}, handle.get_stream()); rmm::device_uvector nbr_intersection_indices(size_t{0}, handle.get_stream()); - [[maybe_unused]] auto nbr_intersection_properties0 = + [[maybe_unused]] auto nbr_intersection_e_property_values0 = cugraph::detail::allocate_optional_dataframe_buffer( 0, handle.get_stream()); - [[maybe_unused]] auto nbr_intersection_properties1 = + [[maybe_unused]] auto nbr_intersection_e_property_values1 = cugraph::detail::allocate_optional_dataframe_buffer( 0, handle.get_stream()); @@ -1116,15 +1255,19 @@ nbr_intersection(raft::handle_t const& handle, edge_partition_nbr_intersection_sizes.reserve(graph_view.number_of_local_edge_partitions()); edge_partition_nbr_intersection_indices.reserve(graph_view.number_of_local_edge_partitions()); - [[maybe_unused]] optional_nbr_intersected_edge_partitions_t - edge_partition_nbr_intersection_property0{}; - [[maybe_unused]] optional_nbr_intersected_edge_partitions_t - edge_partition_nbr_intersection_property1{}; + [[maybe_unused]] std::conditional_t, + std::vector>, + std::byte /* dummy */> + edge_partition_nbr_intersection_e_property_values0{}; + [[maybe_unused]] std::conditional_t, + std::vector>, + std::byte /* dummy */> + edge_partition_nbr_intersection_e_property_values1{}; if constexpr (!std::is_same_v) { - edge_partition_nbr_intersection_property0.reserve( + edge_partition_nbr_intersection_e_property_values0.reserve( graph_view.number_of_local_edge_partitions()); - edge_partition_nbr_intersection_property1.reserve( + edge_partition_nbr_intersection_e_property_values1.reserve( graph_view.number_of_local_edge_partitions()); } @@ -1144,11 +1287,11 @@ nbr_intersection(raft::handle_t const& handle, rmm::device_uvector rx_v_pair_nbr_intersection_indices(size_t{0}, handle.get_stream()); - [[maybe_unused]] auto rx_v_pair_nbr_intersection_properties0 = + [[maybe_unused]] auto rx_v_pair_nbr_intersection_e_property_values0 = cugraph::detail::allocate_optional_dataframe_buffer( 0, handle.get_stream()); - [[maybe_unused]] auto rx_v_pair_nbr_intersection_properties1 = + [[maybe_unused]] auto rx_v_pair_nbr_intersection_e_property_values1 = cugraph::detail::allocate_optional_dataframe_buffer( 0, handle.get_stream()); @@ -1174,9 +1317,15 @@ nbr_intersection(raft::handle_t const& handle, auto edge_partition = edge_partition_device_view_t( graph_view.local_edge_partition_view(i)); - auto edge_partition_e_value_input = edge_partition_e_input_device_view_t(edge_value_input, i); + auto edge_partition_e_mask = + edge_mask_view + ? thrust::make_optional< + detail::edge_partition_edge_property_device_view_t>( + *edge_mask_view, i) + : thrust::nullopt; + auto segment_offsets = graph_view.local_edge_partition_segment_offsets(i); rx_v_pair_nbr_intersection_sizes.resize( @@ -1193,11 +1342,12 @@ nbr_intersection(raft::handle_t const& handle, rx_v_pair_nbr_intersection_sizes.begin(), pick_min_degree_t{ nullptr, - raft::device_span(), + raft::device_span(), second_element_to_idx_map, - raft::device_span((*major_nbr_offsets).data(), + raft::device_span((*major_nbr_offsets).data(), (*major_nbr_offsets).size()), - edge_partition}); + edge_partition, + edge_partition_e_mask}); } else { CUGRAPH_FAIL("unimplemented."); } @@ -1215,25 +1365,30 @@ nbr_intersection(raft::handle_t const& handle, rx_v_pair_nbr_intersection_offsets.back_element(handle.get_stream()), handle.get_stream()); - optional_property_buffer_view_t rx_v_pair_optional_nbr_intersection_properties0{}; - optional_property_buffer_view_t rx_v_pair_optional_nbr_intersection_properties1{}; + optional_property_buffer_mutable_view_t + rx_v_pair_optional_nbr_intersection_e_property_values0{}; + optional_property_buffer_mutable_view_t + rx_v_pair_optional_nbr_intersection_e_property_values1{}; if constexpr (!std::is_same_v) { - rx_v_pair_nbr_intersection_properties0.resize(rx_v_pair_nbr_intersection_indices.size(), - handle.get_stream()); - rx_v_pair_nbr_intersection_properties1.resize(rx_v_pair_nbr_intersection_indices.size(), - handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values0.resize( + rx_v_pair_nbr_intersection_indices.size(), handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values1.resize( + rx_v_pair_nbr_intersection_indices.size(), handle.get_stream()); - rx_v_pair_optional_nbr_intersection_properties0 = - raft::device_span(rx_v_pair_nbr_intersection_properties0.data(), - rx_v_pair_nbr_intersection_properties0.size()); + rx_v_pair_optional_nbr_intersection_e_property_values0 = + rx_v_pair_nbr_intersection_e_property_values0.data(); - rx_v_pair_optional_nbr_intersection_properties1 = - raft::device_span(rx_v_pair_nbr_intersection_properties1.data(), - rx_v_pair_nbr_intersection_properties1.size()); + rx_v_pair_optional_nbr_intersection_e_property_values1 = + rx_v_pair_nbr_intersection_e_property_values1.data(); } if (intersect_minor_nbr[0] && intersect_minor_nbr[1]) { + optional_property_buffer_view_t optional_major_e_property_values{}; + if constexpr (!std::is_same_v) { + optional_major_e_property_values = major_e_property_values.data(); + } + auto second_element_to_idx_map = detail::kv_cuco_store_find_device_view_t((*major_to_idx_map_ptr)->view()); thrust::tabulate( @@ -1248,28 +1403,29 @@ nbr_intersection(raft::handle_t const& handle, edge_t, edge_partition_e_input_device_view_t, optional_property_buffer_view_t, + optional_property_buffer_mutable_view_t, true>{nullptr, - raft::device_span(), + raft::device_span(), raft::device_span(), optional_property_buffer_view_t{}, second_element_to_idx_map, - raft::device_span((*major_nbr_offsets).data(), + raft::device_span((*major_nbr_offsets).data(), (*major_nbr_offsets).size()), raft::device_span((*major_nbr_indices).data(), (*major_nbr_indices).size()), - optional_major_nbr_properties, + optional_major_e_property_values, edge_partition, edge_partition_e_value_input, + edge_partition_e_mask, get_dataframe_buffer_begin(vertex_pair_buffer), raft::device_span(rx_v_pair_nbr_intersection_offsets.data(), rx_v_pair_nbr_intersection_offsets.size()), raft::device_span(rx_v_pair_nbr_intersection_indices.data(), rx_v_pair_nbr_intersection_indices.size()), - rx_v_pair_optional_nbr_intersection_properties0, - rx_v_pair_optional_nbr_intersection_properties1, + rx_v_pair_optional_nbr_intersection_e_property_values0, + rx_v_pair_optional_nbr_intersection_e_property_values1, invalid_vertex_id::value}); - } else { CUGRAPH_FAIL("unimplemented."); } @@ -1284,31 +1440,31 @@ nbr_intersection(raft::handle_t const& handle, handle.get_stream()); rx_v_pair_nbr_intersection_indices.shrink_to_fit(handle.get_stream()); } else { - auto common_nbr_and_properties_begin = thrust::make_zip_iterator( + auto common_nbr_and_e_property_values_begin = thrust::make_zip_iterator( thrust::make_tuple(rx_v_pair_nbr_intersection_indices.begin(), - rx_v_pair_nbr_intersection_properties0.begin(), - rx_v_pair_nbr_intersection_properties1.begin())); + rx_v_pair_nbr_intersection_e_property_values0.begin(), + rx_v_pair_nbr_intersection_e_property_values1.begin())); auto last = thrust::remove_if( handle.get_thrust_policy(), - common_nbr_and_properties_begin, - common_nbr_and_properties_begin + rx_v_pair_nbr_intersection_indices.size(), + common_nbr_and_e_property_values_begin, + common_nbr_and_e_property_values_begin + rx_v_pair_nbr_intersection_indices.size(), [] __device__(auto nbr_p0_p1) { return thrust::get<0>(nbr_p0_p1) == invalid_vertex_id::value; }); rx_v_pair_nbr_intersection_indices.resize( - thrust::distance(common_nbr_and_properties_begin, last), handle.get_stream()); + thrust::distance(common_nbr_and_e_property_values_begin, last), handle.get_stream()); rx_v_pair_nbr_intersection_indices.shrink_to_fit(handle.get_stream()); - rx_v_pair_nbr_intersection_properties0.resize(rx_v_pair_nbr_intersection_indices.size(), - handle.get_stream()); - rx_v_pair_nbr_intersection_properties0.shrink_to_fit(handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values0.resize( + rx_v_pair_nbr_intersection_indices.size(), handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values0.shrink_to_fit(handle.get_stream()); - rx_v_pair_nbr_intersection_properties1.resize(rx_v_pair_nbr_intersection_indices.size(), - handle.get_stream()); - rx_v_pair_nbr_intersection_properties1.shrink_to_fit(handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values1.resize( + rx_v_pair_nbr_intersection_indices.size(), handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values1.shrink_to_fit(handle.get_stream()); } thrust::inclusive_scan(handle.get_thrust_policy(), @@ -1427,11 +1583,11 @@ nbr_intersection(raft::handle_t const& handle, rmm::device_uvector combined_nbr_intersection_indices(size_t{0}, handle.get_stream()); - [[maybe_unused]] auto combined_nbr_intersection_properties0 = + [[maybe_unused]] auto combined_nbr_intersection_e_property_values0 = cugraph::detail::allocate_optional_dataframe_buffer( size_t{0}, handle.get_stream()); - [[maybe_unused]] auto combined_nbr_intersection_properties1 = + [[maybe_unused]] auto combined_nbr_intersection_e_property_values1 = cugraph::detail::allocate_optional_dataframe_buffer( size_t{0}, handle.get_stream()); @@ -1470,47 +1626,47 @@ nbr_intersection(raft::handle_t const& handle, combined_nbr_intersection_indices.resize(gathered_nbr_intersection_indices.size(), handle.get_stream()); - [[maybe_unused]] auto gathered_nbr_intersection_properties0 = + [[maybe_unused]] auto gathered_nbr_intersection_e_property_values0 = cugraph::detail::allocate_optional_dataframe_buffer( rx_displacements.back() + gathered_nbr_intersection_index_rx_counts.back(), handle.get_stream()); - [[maybe_unused]] auto gathered_nbr_intersection_properties1 = + [[maybe_unused]] auto gathered_nbr_intersection_e_property_values1 = cugraph::detail::allocate_optional_dataframe_buffer( rx_displacements.back() + gathered_nbr_intersection_index_rx_counts.back(), handle.get_stream()); if constexpr (!std::is_same_v) { device_multicast_sendrecv(minor_comm, - rx_v_pair_nbr_intersection_properties0.begin(), + rx_v_pair_nbr_intersection_e_property_values0.begin(), rx_v_pair_nbr_intersection_index_tx_counts, tx_displacements, ranks, - gathered_nbr_intersection_properties0.begin(), + gathered_nbr_intersection_e_property_values0.begin(), gathered_nbr_intersection_index_rx_counts, rx_displacements, ranks, handle.get_stream()); - rx_v_pair_nbr_intersection_properties0.resize(size_t{0}, handle.get_stream()); - rx_v_pair_nbr_intersection_properties0.shrink_to_fit(handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values0.resize(size_t{0}, handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values0.shrink_to_fit(handle.get_stream()); - combined_nbr_intersection_properties0.resize(gathered_nbr_intersection_properties0.size(), - handle.get_stream()); + combined_nbr_intersection_e_property_values0.resize( + gathered_nbr_intersection_e_property_values0.size(), handle.get_stream()); device_multicast_sendrecv(minor_comm, - rx_v_pair_nbr_intersection_properties1.begin(), + rx_v_pair_nbr_intersection_e_property_values1.begin(), rx_v_pair_nbr_intersection_index_tx_counts, tx_displacements, ranks, - gathered_nbr_intersection_properties1.begin(), + gathered_nbr_intersection_e_property_values1.begin(), gathered_nbr_intersection_index_rx_counts, rx_displacements, ranks, handle.get_stream()); - rx_v_pair_nbr_intersection_properties1.resize(size_t{0}, handle.get_stream()); - rx_v_pair_nbr_intersection_properties1.shrink_to_fit(handle.get_stream()); - combined_nbr_intersection_properties1.resize(gathered_nbr_intersection_properties1.size(), - handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values1.resize(size_t{0}, handle.get_stream()); + rx_v_pair_nbr_intersection_e_property_values1.shrink_to_fit(handle.get_stream()); + combined_nbr_intersection_e_property_values1.resize( + gathered_nbr_intersection_e_property_values1.size(), handle.get_stream()); } if constexpr (!std::is_same_v) { @@ -1518,7 +1674,10 @@ nbr_intersection(raft::handle_t const& handle, handle.get_thrust_policy(), thrust::make_counting_iterator(size_t{0}), thrust::make_counting_iterator(rx_v_pair_counts[minor_comm_rank]), - gatherv_indices_t{ + gatherv_indices_t{ rx_v_pair_counts[minor_comm_rank], minor_comm_size, raft::device_span(gathered_nbr_intersection_offsets.data(), @@ -1529,25 +1688,19 @@ nbr_intersection(raft::handle_t const& handle, combined_nbr_intersection_offsets.size()), raft::device_span(combined_nbr_intersection_indices.data(), combined_nbr_intersection_indices.size()), - raft::device_span( - gathered_nbr_intersection_properties0.data(), - gathered_nbr_intersection_properties0.size()), - raft::device_span( - gathered_nbr_intersection_properties1.data(), - gathered_nbr_intersection_properties1.size()), - raft::device_span( - combined_nbr_intersection_properties0.data(), - combined_nbr_intersection_properties0.size()), - raft::device_span( - combined_nbr_intersection_properties1.data(), - combined_nbr_intersection_properties1.size())}); - + gathered_nbr_intersection_e_property_values0.data(), + gathered_nbr_intersection_e_property_values1.data(), + combined_nbr_intersection_e_property_values0.data(), + combined_nbr_intersection_e_property_values1.data()}); } else { thrust::for_each( handle.get_thrust_policy(), thrust::make_counting_iterator(size_t{0}), thrust::make_counting_iterator(rx_v_pair_counts[minor_comm_rank]), - gatherv_indices_t{ + gatherv_indices_t{ rx_v_pair_counts[minor_comm_rank], minor_comm_size, raft::device_span(gathered_nbr_intersection_offsets.data(), @@ -1567,10 +1720,10 @@ nbr_intersection(raft::handle_t const& handle, edge_partition_nbr_intersection_indices.push_back( std::move(combined_nbr_intersection_indices)); if constexpr (!std::is_same_v) { - edge_partition_nbr_intersection_property0.push_back( - std::move(combined_nbr_intersection_properties0)); - edge_partition_nbr_intersection_property1.push_back( - std::move(combined_nbr_intersection_properties1)); + edge_partition_nbr_intersection_e_property_values0.push_back( + std::move(combined_nbr_intersection_e_property_values0)); + edge_partition_nbr_intersection_e_property_values1.push_back( + std::move(combined_nbr_intersection_e_property_values1)); } } @@ -1581,8 +1734,10 @@ nbr_intersection(raft::handle_t const& handle, } nbr_intersection_indices.resize(num_nbr_intersection_indices, handle.get_stream()); if constexpr (!std::is_same_v) { - nbr_intersection_properties0.resize(nbr_intersection_indices.size(), handle.get_stream()); - nbr_intersection_properties1.resize(nbr_intersection_indices.size(), handle.get_stream()); + nbr_intersection_e_property_values0.resize(nbr_intersection_indices.size(), + handle.get_stream()); + nbr_intersection_e_property_values1.resize(nbr_intersection_indices.size(), + handle.get_stream()); } size_t size_offset{0}; size_t index_offset{0}; @@ -1599,14 +1754,14 @@ nbr_intersection(raft::handle_t const& handle, if constexpr (!std::is_same_v) { thrust::copy(handle.get_thrust_policy(), - edge_partition_nbr_intersection_property0[i].begin(), - edge_partition_nbr_intersection_property0[i].end(), - nbr_intersection_properties0.begin() + index_offset); + edge_partition_nbr_intersection_e_property_values0[i].begin(), + edge_partition_nbr_intersection_e_property_values0[i].end(), + nbr_intersection_e_property_values0.begin() + index_offset); thrust::copy(handle.get_thrust_policy(), - edge_partition_nbr_intersection_property1[i].begin(), - edge_partition_nbr_intersection_property1[i].end(), - nbr_intersection_properties1.begin() + index_offset); + edge_partition_nbr_intersection_e_property_values1[i].begin(), + edge_partition_nbr_intersection_e_property_values1[i].end(), + nbr_intersection_e_property_values1.begin() + index_offset); } index_offset += edge_partition_nbr_intersection_indices[i].size(); @@ -1619,13 +1774,18 @@ nbr_intersection(raft::handle_t const& handle, size_first, size_first + nbr_intersection_sizes.size(), nbr_intersection_offsets.begin() + 1); - } else { auto edge_partition = edge_partition_device_view_t( graph_view.local_edge_partition_view(size_t{0})); - auto edge_partition_e_value_input = edge_partition_e_input_device_view_t(edge_value_input, 0); + auto edge_partition_e_mask = + edge_mask_view + ? thrust::make_optional< + detail::edge_partition_edge_property_device_view_t>( + *edge_mask_view, 0) + : thrust::nullopt; + rmm::device_uvector nbr_intersection_sizes( input_size, handle.get_stream()); // initially store minimum degrees (upper bound for intersection sizes) @@ -1636,10 +1796,11 @@ nbr_intersection(raft::handle_t const& handle, vertex_pair_first + input_size, nbr_intersection_sizes.begin(), pick_min_degree_t{nullptr, - raft::device_span(), + raft::device_span(), nullptr, - raft::device_span(), - edge_partition}); + raft::device_span(), + edge_partition, + edge_partition_e_mask}); } else { CUGRAPH_FAIL("unimplemented."); } @@ -1656,51 +1817,51 @@ nbr_intersection(raft::handle_t const& handle, nbr_intersection_indices.resize(nbr_intersection_offsets.back_element(handle.get_stream()), handle.get_stream()); - optional_property_buffer_view_t optional_nbr_intersection_properties0{}; - optional_property_buffer_view_t optional_nbr_intersection_properties1{}; + optional_property_buffer_mutable_view_t optional_nbr_intersection_e_property_values0{}; + optional_property_buffer_mutable_view_t optional_nbr_intersection_e_property_values1{}; if constexpr (!std::is_same_v) { - nbr_intersection_properties0.resize(nbr_intersection_indices.size(), handle.get_stream()); - nbr_intersection_properties1.resize(nbr_intersection_indices.size(), handle.get_stream()); - - optional_nbr_intersection_properties0 = raft::device_span( - nbr_intersection_properties0.data(), nbr_intersection_properties0.size()); + nbr_intersection_e_property_values0.resize(nbr_intersection_indices.size(), + handle.get_stream()); + nbr_intersection_e_property_values1.resize(nbr_intersection_indices.size(), + handle.get_stream()); - optional_nbr_intersection_properties1 = raft::device_span( - nbr_intersection_properties1.data(), nbr_intersection_properties1.size()); + optional_nbr_intersection_e_property_values0 = nbr_intersection_e_property_values0.data(); + optional_nbr_intersection_e_property_values1 = nbr_intersection_e_property_values1.data(); } if (intersect_minor_nbr[0] && intersect_minor_nbr[1]) { - thrust::tabulate( - handle.get_thrust_policy(), - nbr_intersection_sizes.begin(), - nbr_intersection_sizes.end(), - copy_intersecting_nbrs_and_update_intersection_size_t{ - nullptr, - raft::device_span(), - raft::device_span(), - optional_property_buffer_view_t{}, - nullptr, - raft::device_span(), - raft::device_span(), - optional_property_buffer_view_t{}, - edge_partition, - edge_partition_e_value_input, - vertex_pair_first, - raft::device_span(nbr_intersection_offsets.data(), - nbr_intersection_offsets.size()), - raft::device_span(nbr_intersection_indices.data(), - nbr_intersection_indices.size()), - optional_nbr_intersection_properties0, - optional_nbr_intersection_properties1, - invalid_vertex_id::value}); + thrust::tabulate(handle.get_thrust_policy(), + nbr_intersection_sizes.begin(), + nbr_intersection_sizes.end(), + copy_intersecting_nbrs_and_update_intersection_size_t< + void*, + void*, + decltype(vertex_pair_first), + vertex_t, + edge_t, + edge_partition_e_input_device_view_t, + optional_property_buffer_view_t, + optional_property_buffer_mutable_view_t, + false>{nullptr, + raft::device_span(), + raft::device_span(), + optional_property_buffer_view_t{}, + nullptr, + raft::device_span(), + raft::device_span(), + optional_property_buffer_view_t{}, + edge_partition, + edge_partition_e_value_input, + edge_partition_e_mask, + vertex_pair_first, + raft::device_span(nbr_intersection_offsets.data(), + nbr_intersection_offsets.size()), + raft::device_span(nbr_intersection_indices.data(), + nbr_intersection_indices.size()), + optional_nbr_intersection_e_property_values0, + optional_nbr_intersection_e_property_values1, + invalid_vertex_id::value}); } else { CUGRAPH_FAIL("unimplemented."); } @@ -1711,14 +1872,14 @@ nbr_intersection(raft::handle_t const& handle, thrust::count_if(handle.get_thrust_policy(), nbr_intersection_indices.begin(), nbr_intersection_indices.end(), - detail::not_equal_t{invalid_vertex_id::value}), + detail::is_not_equal_t{invalid_vertex_id::value}), handle.get_stream()); - [[maybe_unused]] auto tmp_properties0 = + [[maybe_unused]] auto tmp_property_values0 = cugraph::detail::allocate_optional_dataframe_buffer( tmp_indices.size(), handle.get_stream()); - [[maybe_unused]] auto tmp_properties1 = + [[maybe_unused]] auto tmp_property_values1 = cugraph::detail::allocate_optional_dataframe_buffer( tmp_indices.size(), handle.get_stream()); @@ -1737,39 +1898,38 @@ nbr_intersection(raft::handle_t const& handle, nbr_intersection_indices.begin() + num_scanned, nbr_intersection_indices.begin() + num_scanned + this_scan_size, tmp_indices.begin() + num_copied, - detail::not_equal_t{invalid_vertex_id::value}))); + detail::is_not_equal_t{invalid_vertex_id::value}))); } else { - auto zipped_itr_to_indices_and_properties_begin = - thrust::make_zip_iterator(thrust::make_tuple(nbr_intersection_indices.begin(), - nbr_intersection_properties0.begin(), - nbr_intersection_properties1.begin())); + auto zipped_itr_to_indices_and_e_property_values_begin = thrust::make_zip_iterator( + thrust::make_tuple(nbr_intersection_indices.begin(), + nbr_intersection_e_property_values0.begin(), + nbr_intersection_e_property_values1.begin())); auto zipped_itr_to_tmps_begin = thrust::make_zip_iterator(thrust::make_tuple( - tmp_indices.begin(), tmp_properties0.begin(), tmp_properties1.begin())); + tmp_indices.begin(), tmp_property_values0.begin(), tmp_property_values1.begin())); num_copied += static_cast(thrust::distance( zipped_itr_to_tmps_begin + num_copied, - thrust::copy_if(handle.get_thrust_policy(), - zipped_itr_to_indices_and_properties_begin + num_scanned, - zipped_itr_to_indices_and_properties_begin + num_scanned + this_scan_size, - zipped_itr_to_tmps_begin + num_copied, - [] __device__(auto nbr_p0_p1) { - auto nbr = thrust::get<0>(nbr_p0_p1); - auto p0 = thrust::get<1>(nbr_p0_p1); - auto p1 = thrust::get<2>(nbr_p0_p1); - return thrust::get<0>(nbr_p0_p1) != invalid_vertex_id::value; - }))); + thrust::copy_if( + handle.get_thrust_policy(), + zipped_itr_to_indices_and_e_property_values_begin + num_scanned, + zipped_itr_to_indices_and_e_property_values_begin + num_scanned + this_scan_size, + zipped_itr_to_tmps_begin + num_copied, + [] __device__(auto nbr_p0_p1) { + auto nbr = thrust::get<0>(nbr_p0_p1); + auto p0 = thrust::get<1>(nbr_p0_p1); + auto p1 = thrust::get<2>(nbr_p0_p1); + return thrust::get<0>(nbr_p0_p1) != invalid_vertex_id::value; + }))); } num_scanned += this_scan_size; } nbr_intersection_indices = std::move(tmp_indices); if constexpr (!std::is_same_v) { - nbr_intersection_properties0 = std::move(tmp_properties0); - nbr_intersection_properties1 = std::move(tmp_properties1); + nbr_intersection_e_property_values0 = std::move(tmp_property_values0); + nbr_intersection_e_property_values1 = std::move(tmp_property_values1); } - #else - if constexpr (std::is_same_v) { nbr_intersection_indices.resize( thrust::distance(nbr_intersection_indices.begin(), @@ -1780,10 +1940,10 @@ nbr_intersection(raft::handle_t const& handle, handle.get_stream()); } else { nbr_intersection_indices.resize( - thrust::distance(zipped_itr_to_indices_and_properties_begin, + thrust::distance(zipped_itr_to_indices_and_e_property_values_begin, thrust::remove_if(handle.get_thrust_policy(), - zipped_itr_to_indices_and_properties_begin, - zipped_itr_to_indices_and_properties_begin + + zipped_itr_to_indices_and_e_property_values_begin, + zipped_itr_to_indices_and_e_property_values_begin + nbr_intersection_indices.size(), [] __device__(auto nbr_p0_p1) { return thrust::get<0>(nbr_p0_p1) == @@ -1791,8 +1951,10 @@ nbr_intersection(raft::handle_t const& handle, })), handle.get_stream()); - nbr_intersection_properties0.resize(nbr_intersection_indices.size(), handle.get_stream()); - nbr_intersection_properties1.resize(nbr_intersection_indices.size(), handle.get_stream()); + nbr_intersection_e_property_values0.resize(nbr_intersection_indices.size(), + handle.get_stream()); + nbr_intersection_e_property_values1.resize(nbr_intersection_indices.size(), + handle.get_stream()); } #endif @@ -1811,8 +1973,8 @@ nbr_intersection(raft::handle_t const& handle, } else { return std::make_tuple(std::move(nbr_intersection_offsets), std::move(nbr_intersection_indices), - std::move(nbr_intersection_properties0), - std::move(nbr_intersection_properties1)); + std::move(nbr_intersection_e_property_values0), + std::move(nbr_intersection_e_property_values1)); } } diff --git a/cpp/src/prims/detail/optional_dataframe_buffer.hpp b/cpp/src/prims/detail/optional_dataframe_buffer.hpp index dd40e6932e4..62b2245a651 100644 --- a/cpp/src/prims/detail/optional_dataframe_buffer.hpp +++ b/cpp/src/prims/detail/optional_dataframe_buffer.hpp @@ -97,6 +97,7 @@ void shrink_to_fit_optional_dataframe_buffer( { return shrink_to_fit_dataframe_buffer(optional_dataframe_buffer, stream_view); } + } // namespace detail } // namespace cugraph diff --git a/cpp/src/prims/kv_store.cuh b/cpp/src/prims/kv_store.cuh index c46e83aa5da..f17441ad6ab 100644 --- a/cpp/src/prims/kv_store.cuh +++ b/cpp/src/prims/kv_store.cuh @@ -604,7 +604,7 @@ class kv_cuco_store_t { store_value_offsets.begin() /* map */, store_value_offsets.begin() /* stencil */, get_dataframe_buffer_begin(store_values_), - not_equal_t{std::numeric_limits::max()}); + is_not_equal_t{std::numeric_limits::max()}); } } @@ -649,7 +649,7 @@ class kv_cuco_store_t { store_value_offsets.begin() /* map */, store_value_offsets.begin() /* stencil */, get_dataframe_buffer_begin(store_values_), - not_equal_t{std::numeric_limits::max()}); + is_not_equal_t{std::numeric_limits::max()}); } } @@ -695,7 +695,7 @@ class kv_cuco_store_t { store_value_offsets.begin() /* map */, store_value_offsets.begin() /* stencil */, get_dataframe_buffer_begin(store_values_), - not_equal_t{std::numeric_limits::max()}); + is_not_equal_t{std::numeric_limits::max()}); // now perform assigns (for k,v pairs that failed to insert) diff --git a/cpp/src/prims/per_v_pair_transform_dst_nbr_intersection.cuh b/cpp/src/prims/per_v_pair_transform_dst_nbr_intersection.cuh index 640c3c04bfd..201c08325d7 100644 --- a/cpp/src/prims/per_v_pair_transform_dst_nbr_intersection.cuh +++ b/cpp/src/prims/per_v_pair_transform_dst_nbr_intersection.cuh @@ -112,8 +112,8 @@ struct call_intersection_op_t { IntersectionOp intersection_op{}; size_t const* nbr_offsets{nullptr}; typename GraphViewType::vertex_type const* nbr_indices{nullptr}; - EdgeValueInputIterator nbr_intersection_properties0{nullptr}; - EdgeValueInputIterator nbr_intersection_properties1{nullptr}; + EdgeValueInputIterator nbr_intersection_property_values0{nullptr}; + EdgeValueInputIterator nbr_intersection_property_values1{nullptr}; VertexPairIndexIterator major_minor_pair_index_first{}; VertexPairIterator major_minor_pair_first{}; VertexPairValueOutputIterator major_minor_pair_value_output_first{}; @@ -136,20 +136,20 @@ struct call_intersection_op_t { std::conditional_t, raft::device_span, std::byte /* dummy */> - properties0{}; + property_values0{}; std::conditional_t, raft::device_span, std::byte /* dummy */> - properties1{}; + property_values1{}; if constexpr (!std::is_same_v) { - properties0 = raft::device_span( - nbr_intersection_properties0 + nbr_offsets[i], - nbr_intersection_properties0 + +nbr_offsets[i + 1]); - properties1 = raft::device_span( - nbr_intersection_properties1 + nbr_offsets[i], - nbr_intersection_properties1 + +nbr_offsets[i + 1]); + property_values0 = raft::device_span( + nbr_intersection_property_values0 + nbr_offsets[i], + nbr_intersection_property_values0 + +nbr_offsets[i + 1]); + property_values1 = raft::device_span( + nbr_intersection_property_values1 + nbr_offsets[i], + nbr_intersection_property_values1 + +nbr_offsets[i + 1]); } property_t src_prop{}; @@ -174,8 +174,8 @@ struct call_intersection_op_t { dst_prop = *(vertex_property_first + dst_offset); } - *(major_minor_pair_value_output_first + index) = - intersection_op(src, dst, src_prop, dst_prop, intersection, properties0, properties1); + *(major_minor_pair_value_output_first + index) = intersection_op( + src, dst, src_prop, dst_prop, intersection, property_values0, property_values1); } }; @@ -240,8 +240,6 @@ void per_v_pair_transform_dst_nbr_intersection( using edge_property_value_t = typename EdgeValueInputIterator::value_type; using result_t = typename thrust::iterator_traits::value_type; - CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); - if (do_expensive_check) { auto num_invalids = detail::count_invalid_vertex_pairs(handle, graph_view, vertex_pair_first, vertex_pair_last); @@ -384,16 +382,16 @@ void per_v_pair_transform_dst_nbr_intersection( rmm::device_uvector intersection_offsets(size_t{0}, handle.get_stream()); rmm::device_uvector intersection_indices(size_t{0}, handle.get_stream()); - [[maybe_unused]] rmm::device_uvector r_nbr_intersection_properties0( - size_t{0}, handle.get_stream()); - [[maybe_unused]] rmm::device_uvector r_nbr_intersection_properties1( - size_t{0}, handle.get_stream()); + [[maybe_unused]] rmm::device_uvector + r_nbr_intersection_property_values0(size_t{0}, handle.get_stream()); + [[maybe_unused]] rmm::device_uvector + r_nbr_intersection_property_values1(size_t{0}, handle.get_stream()); if constexpr (!std::is_same_v) { std::tie(intersection_offsets, intersection_indices, - r_nbr_intersection_properties0, - r_nbr_intersection_properties1) = + r_nbr_intersection_property_values0, + r_nbr_intersection_property_values1) = detail::nbr_intersection(handle, graph_view, edge_value_input, @@ -422,7 +420,7 @@ void per_v_pair_transform_dst_nbr_intersection( detail::call_intersection_op_t< GraphViewType, decltype(vertex_value_input_for_unique_vertices_first), - typename decltype(r_nbr_intersection_properties0)::const_pointer, + typename decltype(r_nbr_intersection_property_values0)::const_pointer, IntersectionOp, decltype(chunk_vertex_pair_index_first), VertexPairIterator, @@ -433,8 +431,8 @@ void per_v_pair_transform_dst_nbr_intersection( intersection_op, intersection_offsets.data(), intersection_indices.data(), - r_nbr_intersection_properties0.data(), - r_nbr_intersection_properties1.data(), + r_nbr_intersection_property_values0.data(), + r_nbr_intersection_property_values1.data(), chunk_vertex_pair_index_first, vertex_pair_first, vertex_pair_value_output_first}); @@ -445,7 +443,7 @@ void per_v_pair_transform_dst_nbr_intersection( detail::call_intersection_op_t< GraphViewType, VertexValueInputIterator, - typename decltype(r_nbr_intersection_properties0)::const_pointer, + typename decltype(r_nbr_intersection_property_values0)::const_pointer, IntersectionOp, decltype(chunk_vertex_pair_index_first), VertexPairIterator, @@ -456,8 +454,8 @@ void per_v_pair_transform_dst_nbr_intersection( intersection_op, intersection_offsets.data(), intersection_indices.data(), - r_nbr_intersection_properties0.data(), - r_nbr_intersection_properties1.data(), + r_nbr_intersection_property_values0.data(), + r_nbr_intersection_property_values1.data(), chunk_vertex_pair_index_first, vertex_pair_first, vertex_pair_value_output_first}); diff --git a/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh b/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh index e6db21f1c7c..5fee97790f1 100644 --- a/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh +++ b/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh @@ -941,7 +941,7 @@ per_v_random_select_transform_e(raft::handle_t const& handle, minor_comm_ranks.begin(), thrust::make_zip_iterator( thrust::make_tuple(tmp_sample_local_nbr_indices.begin(), tmp_sample_key_indices.begin())), - not_equal_t{-1}); + is_not_equal_t{-1}); sample_local_nbr_indices = std::move(tmp_sample_local_nbr_indices); sample_key_indices = std::move(tmp_sample_key_indices); diff --git a/cpp/src/prims/per_v_transform_reduce_dst_key_aggregated_outgoing_e.cuh b/cpp/src/prims/per_v_transform_reduce_dst_key_aggregated_outgoing_e.cuh index 2e19adc34c4..d4f8606257e 100644 --- a/cpp/src/prims/per_v_transform_reduce_dst_key_aggregated_outgoing_e.cuh +++ b/cpp/src/prims/per_v_transform_reduce_dst_key_aggregated_outgoing_e.cuh @@ -336,8 +336,14 @@ void per_v_transform_reduce_dst_key_aggregated_outgoing_e( if (edge_partition.number_of_edges() > 0) { auto segment_offsets = graph_view.local_edge_partition_segment_offsets(i); - detail::decompress_edge_partition_to_fill_edgelist_majors( - handle, edge_partition, tmp_majors.data(), segment_offsets); + detail::decompress_edge_partition_to_fill_edgelist_majors( + handle, + edge_partition, + std::nullopt, + raft::device_span(tmp_majors.data(), tmp_majors.size()), + segment_offsets); auto minor_key_first = thrust::make_transform_iterator( edge_partition.indices(), diff --git a/cpp/src/prims/per_v_transform_reduce_incoming_outgoing_e.cuh b/cpp/src/prims/per_v_transform_reduce_incoming_outgoing_e.cuh index 1349454f5b6..0b6c6a554bb 100644 --- a/cpp/src/prims/per_v_transform_reduce_incoming_outgoing_e.cuh +++ b/cpp/src/prims/per_v_transform_reduce_incoming_outgoing_e.cuh @@ -513,24 +513,20 @@ void per_v_transform_reduce_e(raft::handle_t const& handle, static_assert(is_arithmetic_or_thrust_tuple_of_arithmetic::value); - [[maybe_unused]] std::conditional_t, - edge_dst_property_t> - minor_tmp_buffer(handle); // relevant only when (GraphViewType::is_multi_gpu && !update_major + using minor_tmp_buffer_type = std::conditional_t, + edge_dst_property_t>; + [[maybe_unused]] std::unique_ptr minor_tmp_buffer{}; if constexpr (GraphViewType::is_multi_gpu && !update_major) { - if constexpr (GraphViewType::is_storage_transposed) { - minor_tmp_buffer = edge_src_property_t(handle, graph_view); - } else { - minor_tmp_buffer = edge_dst_property_t(handle, graph_view); - } + minor_tmp_buffer = std::make_unique(handle, graph_view); } using edge_partition_minor_output_device_view_t = - std::conditional_t>; + decltype(minor_tmp_buffer->mutable_view().value_first())>, + void /* dummy */>; if constexpr (update_major) { size_t partition_idx = 0; @@ -549,7 +545,7 @@ void per_v_transform_reduce_e(raft::handle_t const& handle, } else { if constexpr (GraphViewType::is_multi_gpu) { auto minor_init = init; - auto view = minor_tmp_buffer.view(); + auto view = minor_tmp_buffer->view(); if (view.keys()) { // defer applying the initial value to the end as minor_tmp_buffer may not // store values for the entire minor range minor_init = ReduceOp::identity_element; @@ -558,7 +554,7 @@ void per_v_transform_reduce_e(raft::handle_t const& handle, auto const major_comm_rank = major_comm.get_rank(); minor_init = (major_comm_rank == 0) ? init : ReduceOp::identity_element; } - fill_edge_minor_property(handle, graph_view, minor_init, minor_tmp_buffer.mutable_view()); + fill_edge_minor_property(handle, graph_view, minor_init, minor_tmp_buffer->mutable_view()); } else { thrust::fill(handle.get_thrust_policy(), vertex_value_output_first, @@ -699,7 +695,7 @@ void per_v_transform_reduce_e(raft::handle_t const& handle, if constexpr (update_major) { output_buffer = major_buffer_first; } else { - output_buffer = edge_partition_minor_output_device_view_t(minor_tmp_buffer.mutable_view()); + output_buffer = edge_partition_minor_output_device_view_t(minor_tmp_buffer->mutable_view()); } } else { output_buffer = vertex_value_output_first; @@ -913,7 +909,7 @@ void per_v_transform_reduce_e(raft::handle_t const& handle, auto const minor_comm_rank = minor_comm.get_rank(); auto const minor_comm_size = minor_comm.get_size(); - auto view = minor_tmp_buffer.view(); + auto view = minor_tmp_buffer->view(); if (view.keys()) { // applying the initial value is deferred to here vertex_t max_vertex_partition_size{0}; for (int i = 0; i < major_comm_size; ++i) { diff --git a/cpp/src/prims/transform_e.cuh b/cpp/src/prims/transform_e.cuh index 7950df58a3e..edacdc8a970 100644 --- a/cpp/src/prims/transform_e.cuh +++ b/cpp/src/prims/transform_e.cuh @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,70 @@ namespace cugraph { +namespace detail { + +int32_t constexpr transform_e_kernel_block_size = 512; + +template +__global__ void transform_e_packed_bool( + edge_partition_device_view_t edge_partition, + EdgePartitionSrcValueInputWrapper edge_partition_src_value_input, + EdgePartitionDstValueInputWrapper edge_partition_dst_value_input, + EdgePartitionEdgeValueInputWrapper edge_partition_e_value_input, + EdgePartitionEdgeValueOutputWrapper edge_partition_e_value_output, + EdgeOp e_op) +{ + static_assert(EdgePartitionEdgeValueOutputWrapper::is_packed_bool); + static_assert(raft::warp_size() == packed_bools_per_word()); + + using edge_t = typename GraphViewType::edge_type; + + auto const tid = threadIdx.x + blockIdx.x * blockDim.x; + static_assert(transform_e_kernel_block_size % raft::warp_size() == 0); + auto const lane_id = tid % raft::warp_size(); + auto idx = static_cast(packed_bool_offset(tid)); + + auto num_edges = edge_partition.number_of_edges(); + while (idx < static_cast(packed_bool_size(num_edges))) { + auto local_edge_idx = + idx * static_cast(packed_bools_per_word()) + static_cast(lane_id); + uint32_t mask{0}; + int predicate{0}; + if (local_edge_idx < num_edges) { + auto major_idx = edge_partition.major_idx_from_local_edge_idx_nocheck(local_edge_idx); + auto major = edge_partition.major_from_major_idx_nocheck(major_idx); + auto major_offset = edge_partition.major_offset_from_major_nocheck(major); + auto minor = *(edge_partition.indices() + local_edge_idx); + auto minor_offset = edge_partition.minor_offset_from_minor_nocheck(minor); + + auto src = GraphViewType::is_storage_transposed ? minor : major; + auto dst = GraphViewType::is_storage_transposed ? major : minor; + auto src_offset = GraphViewType::is_storage_transposed ? minor_offset : major_offset; + auto dst_offset = GraphViewType::is_storage_transposed ? major_offset : minor_offset; + predicate = e_op(src, + dst, + edge_partition_src_value_input.get(src_offset), + edge_partition_dst_value_input.get(dst_offset), + edge_partition_e_value_input.get(local_edge_idx)) + ? int{1} + : int{0}; + } + mask = __ballot_sync(uint32_t{0xffffffff}, predicate); + if (lane_id == 0) { *(edge_partition_e_value_output.value_first() + idx) = mask; } + + idx += static_cast(gridDim.x * (blockDim.x / raft::warp_size())); + } +} + +} // namespace detail + /** * @brief Iterate over the entire set of edges and update edge property values. * @@ -84,9 +149,103 @@ void transform_e(raft::handle_t const& handle, EdgeValueOutputWrapper edge_value_output, bool do_expensive_check = false) { - // CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); + using vertex_t = typename GraphViewType::vertex_type; + using edge_t = typename GraphViewType::edge_type; - CUGRAPH_FAIL("unimplemented."); + using edge_partition_src_input_device_view_t = std::conditional_t< + std::is_same_v, + detail::edge_partition_endpoint_dummy_property_device_view_t, + detail::edge_partition_endpoint_property_device_view_t< + vertex_t, + typename EdgeSrcValueInputWrapper::value_iterator, + typename EdgeSrcValueInputWrapper::value_type>>; + using edge_partition_dst_input_device_view_t = std::conditional_t< + std::is_same_v, + detail::edge_partition_endpoint_dummy_property_device_view_t, + detail::edge_partition_endpoint_property_device_view_t< + vertex_t, + typename EdgeDstValueInputWrapper::value_iterator, + typename EdgeDstValueInputWrapper::value_type>>; + using edge_partition_e_input_device_view_t = std::conditional_t< + std::is_same_v, + detail::edge_partition_edge_dummy_property_device_view_t, + detail::edge_partition_edge_property_device_view_t< + edge_t, + typename EdgeValueInputWrapper::value_iterator, + typename EdgeValueInputWrapper::value_type>>; + using edge_partition_e_output_device_view_t = detail::edge_partition_edge_property_device_view_t< + edge_t, + typename EdgeValueOutputWrapper::value_iterator, + typename EdgeValueOutputWrapper::value_type>; + + CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); + + for (size_t i = 0; i < graph_view.number_of_local_edge_partitions(); ++i) { + auto edge_partition = + edge_partition_device_view_t( + graph_view.local_edge_partition_view(i)); + + edge_partition_src_input_device_view_t edge_partition_src_value_input{}; + edge_partition_dst_input_device_view_t edge_partition_dst_value_input{}; + if constexpr (GraphViewType::is_storage_transposed) { + edge_partition_src_value_input = edge_partition_src_input_device_view_t(edge_src_value_input); + edge_partition_dst_value_input = + edge_partition_dst_input_device_view_t(edge_dst_value_input, i); + } else { + edge_partition_src_value_input = + edge_partition_src_input_device_view_t(edge_src_value_input, i); + edge_partition_dst_value_input = edge_partition_dst_input_device_view_t(edge_dst_value_input); + } + auto edge_partition_e_value_input = edge_partition_e_input_device_view_t(edge_value_input, i); + auto edge_partition_e_value_output = + edge_partition_e_output_device_view_t(edge_value_output, i); + + auto num_edges = edge_partition.number_of_edges(); + if constexpr (edge_partition_e_output_device_view_t::has_packed_bool_element) { + static_assert(edge_partition_e_output_device_view_t::is_packed_bool, + "unimplemented for thrust::tuple types."); + if (edge_partition.number_of_edges() > edge_t{0}) { + raft::grid_1d_thread_t update_grid(num_edges, + detail::transform_e_kernel_block_size, + handle.get_device_properties().maxGridSize[0]); + detail::transform_e_packed_bool + <<>>( + edge_partition, + edge_partition_src_value_input, + edge_partition_dst_value_input, + edge_partition_e_value_input, + edge_partition_e_value_output, + e_op); + } + } else { + thrust::transform( + handle.get_thrust_policy(), + thrust::make_counting_iterator(edge_t{0}), + thrust::make_counting_iterator(num_edges), + edge_partition_e_value_output.value_first(), + [e_op, + edge_partition, + edge_partition_src_value_input, + edge_partition_dst_value_input, + edge_partition_e_value_input] __device__(edge_t i) { + auto major_idx = edge_partition.major_idx_from_local_edge_idx_nocheck(i); + auto major = edge_partition.major_from_major_idx_nocheck(major_idx); + auto major_offset = edge_partition.major_offset_from_major_nocheck(major); + auto minor = *(edge_partition.indices() + i); + auto minor_offset = edge_partition.minor_offset_from_minor_nocheck(minor); + + auto src = GraphViewType::is_storage_transposed ? minor : major; + auto dst = GraphViewType::is_storage_transposed ? major : minor; + auto src_offset = GraphViewType::is_storage_transposed ? minor_offset : major_offset; + auto dst_offset = GraphViewType::is_storage_transposed ? major_offset : minor_offset; + return e_op(src, + dst, + edge_partition_src_value_input.get(src_offset), + edge_partition_dst_value_input.get(dst_offset), + edge_partition_e_value_input.get(i)); + }); + } + } } /** @@ -177,7 +336,7 @@ void transform_e(raft::handle_t const& handle, typename EdgeValueOutputWrapper::value_iterator, typename EdgeValueOutputWrapper::value_type>; - // CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); + CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); auto major_first = GraphViewType::is_storage_transposed ? edge_list.dst_begin() : edge_list.src_begin(); diff --git a/cpp/src/prims/transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v.cuh b/cpp/src/prims/transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v.cuh index f773a102959..bcf7606c423 100644 --- a/cpp/src/prims/transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v.cuh +++ b/cpp/src/prims/transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v.cuh @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -269,10 +270,17 @@ void transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v( vertex_value_output_first + graph_view.local_vertex_partition_range_size(), init); + auto edge_mask_view = graph_view.edge_mask_view(); for (size_t i = 0; i < graph_view.number_of_local_edge_partitions(); ++i) { auto edge_partition = edge_partition_device_view_t( graph_view.local_edge_partition_view(i)); + auto edge_partition_e_mask = + edge_mask_view + ? std::make_optional< + detail::edge_partition_edge_property_device_view_t>( + *edge_mask_view, i) + : std::nullopt; edge_partition_src_input_device_view_t edge_partition_src_value_input{}; edge_partition_dst_input_device_view_t edge_partition_dst_value_input{}; @@ -286,22 +294,29 @@ void transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v( edge_partition_dst_value_input = edge_partition_dst_input_device_view_t(edge_dst_value_input); } - rmm::device_uvector majors(edge_partition.number_of_edges(), handle.get_stream()); + rmm::device_uvector majors( + edge_partition_e_mask + ? detail::count_set_bits( + handle, (*edge_partition_e_mask).value_first(), edge_partition.number_of_edges()) + : static_cast(edge_partition.number_of_edges()), + handle.get_stream()); rmm::device_uvector minors(majors.size(), handle.get_stream()); auto segment_offsets = graph_view.local_edge_partition_segment_offsets(i); detail::decompress_edge_partition_to_edgelist(handle, - edge_partition, - std::nullopt, - std::nullopt, - majors.data(), - minors.data(), - std::nullopt, - std::nullopt, - segment_offsets); + GraphViewType::is_multi_gpu>( + handle, + edge_partition, + std::nullopt, + std::nullopt, + edge_partition_e_mask, + raft::device_span(majors.data(), majors.size()), + raft::device_span(minors.data(), minors.size()), + std::nullopt, + std::nullopt, + segment_offsets); auto vertex_pair_first = thrust::make_zip_iterator(thrust::make_tuple(majors.begin(), minors.begin())); diff --git a/cpp/src/structure/coarsen_graph_impl.cuh b/cpp/src/structure/coarsen_graph_impl.cuh index b8dc28d563e..0704bc6f23b 100644 --- a/cpp/src/structure/coarsen_graph_impl.cuh +++ b/cpp/src/structure/coarsen_graph_impl.cuh @@ -164,16 +164,18 @@ decompress_edge_partition_to_relabeled_and_grouped_and_coarsened_edgelist( ? std::make_optional>( edgelist_majors.size(), handle.get_stream()) : std::nullopt; - detail::decompress_edge_partition_to_edgelist( + detail::decompress_edge_partition_to_edgelist( handle, edge_partition, edge_partition_weight_view, - std::optional>{ - std::nullopt}, - edgelist_majors.data(), - edgelist_minors.data(), - edgelist_weights ? std::optional{(*edgelist_weights).data()} : std::nullopt, - std::optional{std::nullopt}, + std::nullopt, + std::nullopt, + raft::device_span(edgelist_majors.data(), edgelist_majors.size()), + raft::device_span(edgelist_minors.data(), edgelist_minors.size()), + edgelist_weights ? std::make_optional>((*edgelist_weights).data(), + (*edgelist_weights).size()) + : std::nullopt, + std::nullopt, segment_offsets); auto pair_first = diff --git a/cpp/src/structure/create_graph_from_edgelist_impl.cuh b/cpp/src/structure/create_graph_from_edgelist_impl.cuh index 0d4b12a3e38..8dd587e1661 100644 --- a/cpp/src/structure/create_graph_from_edgelist_impl.cuh +++ b/cpp/src/structure/create_graph_from_edgelist_impl.cuh @@ -510,7 +510,18 @@ create_graph_from_edgelist_impl( auto use_dcs = num_segments_per_vertex_partition > (detail::num_sparse_segments_per_vertex_partition + 2); - // 4. compress edge list (COO) to CSR (or CSC) or CSR + DCSR (CSC + DCSC) hybrid + // 4. sort and compress edge list (COO) to CSR (or CSC) or CSR + DCSR (CSC + DCSC) hybrid + + auto total_global_mem = handle.get_device_properties().totalGlobalMem; + size_t element_size = sizeof(vertex_t) * 2; + if (edgelist_weights) { element_size += sizeof(weight_t); } + if (edgelist_edge_ids) { element_size += sizeof(edge_id_t); } + if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } + auto constexpr mem_frugal_ratio = + 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the + // total_global_mem, switch to the memory frugal approach + auto mem_frugal_threshold = + static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); std::vector> edge_partition_offsets; std::vector> edge_partition_indices; @@ -559,154 +570,139 @@ create_graph_from_edgelist_impl( if (edgelist_weights) { if (edgelist_edge_ids) { if (edgelist_edge_types) { - auto edge_value_first = - thrust::make_zip_iterator((*edge_partition_edgelist_weights)[i].begin(), - (*edge_partition_edgelist_edge_ids)[i].begin(), - (*edge_partition_edgelist_edge_types)[i].begin()); std::forward_as_tuple( offsets, indices, std::tie(weights, edge_ids, edge_types), dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::make_tuple(std::move((*edge_partition_edgelist_weights)[i]), + std::move((*edge_partition_edgelist_edge_ids)[i]), + std::move((*edge_partition_edgelist_edge_types)[i])), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } else { - auto edge_value_first = - thrust::make_zip_iterator((*edge_partition_edgelist_weights)[i].begin(), - (*edge_partition_edgelist_edge_ids)[i].begin()); std::forward_as_tuple(offsets, indices, std::tie(weights, edge_ids), dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::make_tuple(std::move((*edge_partition_edgelist_weights)[i]), + std::move((*edge_partition_edgelist_edge_ids)[i])), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } } else { if (edgelist_edge_types) { - auto edge_value_first = - thrust::make_zip_iterator((*edge_partition_edgelist_weights)[i].begin(), - (*edge_partition_edgelist_edge_types)[i].begin()); std::forward_as_tuple(offsets, indices, std::tie(weights, edge_types), dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::make_tuple(std::move((*edge_partition_edgelist_weights)[i]), + std::move((*edge_partition_edgelist_edge_types)[i])), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } else { - auto edge_value_first = (*edge_partition_edgelist_weights)[i].begin(); std::forward_as_tuple(offsets, indices, weights, dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::move((*edge_partition_edgelist_weights)[i]), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } } } else { if (edgelist_edge_ids) { if (edgelist_edge_types) { - auto edge_value_first = - thrust::make_zip_iterator((*edge_partition_edgelist_edge_ids)[i].begin(), - (*edge_partition_edgelist_edge_types)[i].begin()); std::forward_as_tuple( offsets, indices, std::tie(edge_ids, edge_types), dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::make_tuple(std::move((*edge_partition_edgelist_edge_ids)[i]), + std::move((*edge_partition_edgelist_edge_types)[i])), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } else { - auto edge_value_first = (*edge_partition_edgelist_edge_ids)[i].begin(); std::forward_as_tuple(offsets, indices, edge_ids, dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::move((*edge_partition_edgelist_edge_ids)[i]), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } } else { if (edgelist_edge_types) { - auto edge_value_first = (*edge_partition_edgelist_edge_types)[i].begin(); std::forward_as_tuple(offsets, indices, edge_types, dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), - edge_value_first, + detail::sort_and_compress_edgelist( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), + std::move((*edge_partition_edgelist_edge_types)[i]), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } else { std::forward_as_tuple(offsets, indices, dcs_nzd_vertices) = - detail::compress_edgelist( - edge_partition_edgelist_srcs[i].begin(), - edge_partition_edgelist_srcs[i].end(), - edge_partition_edgelist_dsts[i].begin(), + detail::sort_and_compress_edgelist( + std::move(edge_partition_edgelist_srcs[i]), + std::move(edge_partition_edgelist_dsts[i]), major_range_first, major_hypersparse_first, major_range_last, minor_range_first, minor_range_last, + mem_frugal_threshold, handle.get_stream()); } } } - edge_partition_edgelist_srcs[i].resize(0, handle.get_stream()); - edge_partition_edgelist_srcs[i].shrink_to_fit(handle.get_stream()); - edge_partition_edgelist_dsts[i].resize(0, handle.get_stream()); - edge_partition_edgelist_dsts[i].shrink_to_fit(handle.get_stream()); - if (edge_partition_edgelist_weights) { - (*edge_partition_edgelist_weights)[i].resize(0, handle.get_stream()); - (*edge_partition_edgelist_weights)[i].shrink_to_fit(handle.get_stream()); - } - if (edge_partition_edgelist_edge_ids) { - (*edge_partition_edgelist_edge_ids)[i].resize(0, handle.get_stream()); - (*edge_partition_edgelist_edge_ids)[i].shrink_to_fit(handle.get_stream()); - } - if (edge_partition_edgelist_edge_types) { - (*edge_partition_edgelist_edge_types)[i].resize(0, handle.get_stream()); - (*edge_partition_edgelist_edge_types)[i].shrink_to_fit(handle.get_stream()); - } edge_partition_offsets.push_back(std::move(offsets)); edge_partition_indices.push_back(std::move(indices)); if (edge_partition_weights) { (*edge_partition_weights).push_back(std::move(*weights)); } @@ -954,6 +950,17 @@ create_graph_from_edgelist_impl( // convert edge list (COO) to compressed sparse format (CSR or CSC) + auto total_global_mem = handle.get_device_properties().totalGlobalMem; + size_t element_size = sizeof(vertex_t) * 2; + if (edgelist_weights) { element_size += sizeof(weight_t); } + if (edgelist_edge_ids) { element_size += sizeof(edge_id_t); } + if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } + auto constexpr mem_frugal_ratio = + 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the + // total_global_mem, switch to the memory frugal approach + auto mem_frugal_threshold = + static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); + rmm::device_uvector offsets(size_t{0}, handle.get_stream()); rmm::device_uvector indices(size_t{0}, handle.get_stream()); std::optional> weights{std::nullopt}; @@ -963,202 +970,130 @@ create_graph_from_edgelist_impl( if (edgelist_weights) { if (edgelist_edge_ids) { if (edgelist_edge_types) { - auto edge_value_first = thrust::make_zip_iterator((*edgelist_weights).begin(), - (*edgelist_edge_ids).begin(), - (*edgelist_edge_types).begin()); std::forward_as_tuple(offsets, indices, std::tie(weights, ids, types), std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), + std::move(*edgelist_edge_ids), + std::move(*edgelist_edge_types)), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } else { - auto edge_value_first = - thrust::make_zip_iterator((*edgelist_weights).begin(), (*edgelist_edge_ids).begin()); std::forward_as_tuple(offsets, indices, std::tie(weights, ids), std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), std::move(*edgelist_edge_ids)), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } } else { if (edgelist_edge_types) { - auto edge_value_first = - thrust::make_zip_iterator((*edgelist_weights).begin(), (*edgelist_edge_types).begin()); std::forward_as_tuple(offsets, indices, std::tie(weights, types), std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), std::move(*edgelist_edge_types)), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } else { - auto edge_value_first = (*edgelist_weights).begin(); std::forward_as_tuple(offsets, indices, weights, std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(*edgelist_weights), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } } } else { if (edgelist_edge_ids) { if (edgelist_edge_types) { - auto edge_value_first = - thrust::make_zip_iterator((*edgelist_edge_ids).begin(), (*edgelist_edge_types).begin()); std::forward_as_tuple(offsets, indices, std::tie(ids, types), std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist, + store_transposed>( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_ids), std::move(*edgelist_edge_types)), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } else { - auto edge_value_first = (*edgelist_edge_ids).begin(); std::forward_as_tuple(offsets, indices, ids, std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(*edgelist_edge_ids), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } } else { if (edgelist_edge_types) { - auto edge_value_first = (*edgelist_edge_types).begin(); std::forward_as_tuple(offsets, indices, types, std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - edge_value_first, - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); + detail::sort_and_compress_edgelist( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(*edgelist_edge_types), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } else { std::forward_as_tuple(offsets, indices, std::ignore) = - detail::compress_edgelist(edgelist_srcs.begin(), - edgelist_srcs.end(), - edgelist_dsts.begin(), - vertex_t{0}, - std::optional{std::nullopt}, - num_vertices, - vertex_t{0}, - num_vertices, - handle.get_stream()); - } - } - } - - edgelist_srcs.resize(0, handle.get_stream()); - edgelist_srcs.shrink_to_fit(handle.get_stream()); - edgelist_dsts.resize(0, handle.get_stream()); - edgelist_dsts.shrink_to_fit(handle.get_stream()); - if (edgelist_weights) { - (*edgelist_weights).resize(0, handle.get_stream()); - (*edgelist_weights).shrink_to_fit(handle.get_stream()); - } - if (edgelist_edge_ids) { - (*edgelist_edge_ids).resize(0, handle.get_stream()); - (*edgelist_edge_ids).shrink_to_fit(handle.get_stream()); - } - if (edgelist_edge_types) { - (*edgelist_edge_types).resize(0, handle.get_stream()); - (*edgelist_edge_types).shrink_to_fit(handle.get_stream()); - } - - // segmented sort neighbors - - if (weights) { - if (ids) { - if (types) { - detail::sort_adjacency_list( - handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - thrust::make_zip_iterator((*weights).begin(), (*ids).begin(), (*types).begin())); - } else { - detail::sort_adjacency_list(handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - thrust::make_zip_iterator((*weights).begin(), (*ids).begin())); - } - } else { - if (types) { - detail::sort_adjacency_list( - handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - thrust::make_zip_iterator((*weights).begin(), (*types).begin())); - } else { - detail::sort_adjacency_list(handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - (*weights).begin()); - } - } - } else { - if (ids) { - if (types) { - detail::sort_adjacency_list(handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - thrust::make_zip_iterator((*ids).begin(), (*types).begin())); - } else { - detail::sort_adjacency_list(handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - (*ids).begin()); - } - } else { - if (types) { - detail::sort_adjacency_list(handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end(), - (*types).begin()); - } else { - detail::sort_adjacency_list(handle, - raft::device_span(offsets.data(), offsets.size()), - indices.begin(), - indices.end()); + detail::sort_and_compress_edgelist( + std::move(edgelist_srcs), + std::move(edgelist_dsts), + vertex_t{0}, + std::optional{std::nullopt}, + num_vertices, + vertex_t{0}, + num_vertices, + mem_frugal_threshold, + handle.get_stream()); } } } diff --git a/cpp/src/structure/decompress_to_edgelist_impl.cuh b/cpp/src/structure/decompress_to_edgelist_impl.cuh index d653307c620..e9e84f5bf15 100644 --- a/cpp/src/structure/decompress_to_edgelist_impl.cuh +++ b/cpp/src/structure/decompress_to_edgelist_impl.cuh @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -80,8 +81,11 @@ decompress_to_edgelist_impl( std::vector edgelist_edge_counts(graph_view.number_of_local_edge_partitions(), size_t{0}); for (size_t i = 0; i < edgelist_edge_counts.size(); ++i) { - edgelist_edge_counts[i] = - static_cast(graph_view.number_of_local_edge_partition_edges(i)); + edgelist_edge_counts[i] = graph_view.local_edge_partition_view(i).number_of_edges(); + if (graph_view.has_edge_mask()) { + edgelist_edge_counts[i] = detail::count_set_bits( + handle, (*(graph_view.edge_mask_view())).value_firsts()[i], edgelist_edge_counts[i]); + } } auto number_of_local_edges = std::reduce(edgelist_edge_counts.begin(), edgelist_edge_counts.end()); @@ -97,7 +101,7 @@ decompress_to_edgelist_impl( size_t cur_size{0}; for (size_t i = 0; i < edgelist_edge_counts.size(); ++i) { - detail::decompress_edge_partition_to_edgelist( + detail::decompress_edge_partition_to_edgelist( handle, edge_partition_device_view_t( graph_view.local_edge_partition_view(i)), @@ -110,11 +114,19 @@ decompress_to_edgelist_impl( detail::edge_partition_edge_property_device_view_t>( (*edge_id_view), i) : std::nullopt, - edgelist_majors.data() + cur_size, - edgelist_minors.data() + cur_size, - edgelist_weights ? std::optional{(*edgelist_weights).data() + cur_size} + graph_view.has_edge_mask() + ? std::make_optional< + detail::edge_partition_edge_property_device_view_t>( + *(graph_view.edge_mask_view()), i) + : std::nullopt, + raft::device_span(edgelist_majors.data() + cur_size, edgelist_edge_counts[i]), + raft::device_span(edgelist_minors.data() + cur_size, edgelist_edge_counts[i]), + edgelist_weights ? std::make_optional>( + (*edgelist_weights).data() + cur_size, edgelist_edge_counts[i]) : std::nullopt, - edgelist_ids ? std::optional{(*edgelist_ids).data() + cur_size} : std::nullopt, + edgelist_ids ? std::make_optional>( + (*edgelist_ids).data() + cur_size, edgelist_edge_counts[i]) + : std::nullopt, graph_view.local_edge_partition_segment_offsets(i)); cur_size += edgelist_edge_counts[i]; } @@ -231,8 +243,13 @@ decompress_to_edgelist_impl( if (do_expensive_check) { /* currently, nothing to do */ } - rmm::device_uvector edgelist_majors(graph_view.number_of_local_edge_partition_edges(), - handle.get_stream()); + auto num_edges = graph_view.local_edge_partition_view().number_of_edges(); + if (graph_view.has_edge_mask()) { + num_edges = + detail::count_set_bits(handle, (*(graph_view.edge_mask_view())).value_firsts()[0], num_edges); + } + + rmm::device_uvector edgelist_majors(num_edges, handle.get_stream()); rmm::device_uvector edgelist_minors(edgelist_majors.size(), handle.get_stream()); auto edgelist_weights = edge_weight_view ? std::make_optional>( edgelist_majors.size(), handle.get_stream()) @@ -240,7 +257,7 @@ decompress_to_edgelist_impl( auto edgelist_ids = edge_id_view ? std::make_optional>( edgelist_majors.size(), handle.get_stream()) : std::nullopt; - detail::decompress_edge_partition_to_edgelist( + detail::decompress_edge_partition_to_edgelist( handle, edge_partition_device_view_t( graph_view.local_edge_partition_view()), @@ -253,10 +270,19 @@ decompress_to_edgelist_impl( detail::edge_partition_edge_property_device_view_t>( (*edge_id_view), 0) : std::nullopt, - edgelist_majors.data(), - edgelist_minors.data(), - edgelist_weights ? std::optional{(*edgelist_weights).data()} : std::nullopt, - edgelist_ids ? std::optional{(*edgelist_ids).data()} : std::nullopt, + graph_view.has_edge_mask() + ? std::make_optional< + detail::edge_partition_edge_property_device_view_t>( + *(graph_view.edge_mask_view()), 0) + : std::nullopt, + raft::device_span(edgelist_majors.data(), edgelist_majors.size()), + raft::device_span(edgelist_minors.data(), edgelist_minors.size()), + edgelist_weights ? std::make_optional>((*edgelist_weights).data(), + (*edgelist_weights).size()) + : std::nullopt, + edgelist_ids ? std::make_optional>((*edgelist_ids).data(), + (*edgelist_ids).size()) + : std::nullopt, graph_view.local_edge_partition_segment_offsets()); if (renumber_map) { @@ -294,8 +320,6 @@ decompress_to_edgelist( std::optional> renumber_map, bool do_expensive_check) { - CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); - return decompress_to_edgelist_impl( handle, graph_view, edge_weight_view, edge_id_view, renumber_map, do_expensive_check); } diff --git a/cpp/src/structure/detail/structure_utils.cuh b/cpp/src/structure/detail/structure_utils.cuh index f57b549e1ef..c49b62e4543 100644 --- a/cpp/src/structure/detail/structure_utils.cuh +++ b/cpp/src/structure/detail/structure_utils.cuh @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -47,57 +49,38 @@ namespace cugraph { namespace detail { -template -struct update_edge_t { - raft::device_span offsets{}; - raft::device_span indices{}; - EdgeValueIterator edge_value_first{}; - vertex_t major_range_first{}; - - __device__ void operator()(typename thrust::iterator_traits::value_type e) const - { - auto s = thrust::get<0>(e); - auto d = thrust::get<1>(e); - auto major = store_transposed ? d : s; - auto minor = store_transposed ? s : d; - auto start = offsets[major - major_range_first]; - auto degree = offsets[(major - major_range_first) + 1] - start; - auto idx = - atomicAdd(&indices[start + degree - 1], vertex_t{1}); // use the last element as a counter - // FIXME: we can actually store minor - minor_range_first instead of minor to save memory if - // minor can be larger than 32 bit but minor - minor_range_first fits within 32 bit - indices[start + idx] = minor; // overwrite the counter only if idx == degree - 1 (no race) - if constexpr (!std::is_same_v) { - auto value = thrust::get<2>(e); - *(edge_value_first + (start + idx)) = value; - } - } -}; - template rmm::device_uvector compute_sparse_offsets( VertexIterator edgelist_major_first, VertexIterator edgelist_major_last, typename thrust::iterator_traits::value_type major_range_first, typename thrust::iterator_traits::value_type major_range_last, + bool edgelist_major_sorted, rmm::cuda_stream_view stream_view) { rmm::device_uvector offsets((major_range_last - major_range_first) + 1, stream_view); - thrust::fill(rmm::exec_policy(stream_view), offsets.begin(), offsets.end(), edge_t{0}); - - auto offset_view = raft::device_span(offsets.data(), offsets.size()); - thrust::for_each(rmm::exec_policy(stream_view), - edgelist_major_first, - edgelist_major_last, - [offset_view, major_range_first] __device__(auto v) { - atomicAdd(&offset_view[v - major_range_first], edge_t{1}); - }); - thrust::exclusive_scan( - rmm::exec_policy(stream_view), offsets.begin(), offsets.end(), offsets.begin()); + if (edgelist_major_sorted) { + offsets.set_element_to_zero_async(0, stream_view); + thrust::upper_bound(rmm::exec_policy(stream_view), + edgelist_major_first, + edgelist_major_last, + thrust::make_counting_iterator(major_range_first), + thrust::make_counting_iterator(major_range_last), + offsets.begin() + 1); + } else { + thrust::fill(rmm::exec_policy(stream_view), offsets.begin(), offsets.end(), edge_t{0}); + + auto offset_view = raft::device_span(offsets.data(), offsets.size()); + thrust::for_each(rmm::exec_policy(stream_view), + edgelist_major_first, + edgelist_major_last, + [offset_view, major_range_first] __device__(auto v) { + atomicAdd(&offset_view[v - major_range_first], edge_t{1}); + }); + + thrust::exclusive_scan( + rmm::exec_policy(stream_view), offsets.begin(), offsets.end(), offsets.begin()); + } return offsets; } @@ -156,61 +139,77 @@ std::tuple, rmm::device_uvector> compress_ } // compress edge list (COO) to CSR (or CSC) or CSR + DCSR (CSC + DCSC) hybrid -template -std::tuple< - rmm::device_uvector, - rmm::device_uvector::value_type>, - decltype(allocate_dataframe_buffer::value_type>(size_t{0}, rmm::cuda_stream_view{})), - std::optional::value_type>>> -compress_edgelist( - VertexIterator edgelist_src_first, - VertexIterator edgelist_src_last, - VertexIterator edgelist_dst_first, - EdgeValueIterator edge_value_first, - typename thrust::iterator_traits::value_type major_range_first, - std::optional::value_type> - major_hypersparse_first, - typename thrust::iterator_traits::value_type major_range_last, - typename thrust::iterator_traits::value_type /* minor_range_first */, - typename thrust::iterator_traits::value_type /* minor_range_last */, +template +std::tuple, + rmm::device_uvector, + decltype(allocate_dataframe_buffer(size_t{0}, rmm::cuda_stream_view{})), + std::optional>> +sort_and_compress_edgelist( + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + decltype(allocate_dataframe_buffer(0, rmm::cuda_stream_view{}))&& edgelist_values, + vertex_t major_range_first, + std::optional major_hypersparse_first, + vertex_t major_range_last, + vertex_t /* minor_range_first */, + vertex_t /* minor_range_last */, + size_t mem_frugal_threshold, rmm::cuda_stream_view stream_view) { - using vertex_t = std::remove_cv_t::value_type>; - using edge_value_t = - std::remove_cv_t::value_type>; - - auto number_of_edges = - static_cast(thrust::distance(edgelist_src_first, edgelist_src_last)); - - auto offsets = compute_sparse_offsets( - store_transposed ? edgelist_dst_first : edgelist_src_first, - store_transposed ? edgelist_dst_first + number_of_edges : edgelist_src_last, - major_range_first, - major_range_last, - stream_view); - - rmm::device_uvector indices(number_of_edges, stream_view); - thrust::fill(rmm::exec_policy(stream_view), indices.begin(), indices.end(), vertex_t{0}); - auto values = allocate_dataframe_buffer(number_of_edges, stream_view); - - auto offset_view = raft::device_span(offsets.data(), offsets.size()); - auto index_view = raft::device_span(indices.data(), indices.size()); - auto edge_first = thrust::make_zip_iterator( - thrust::make_tuple(edgelist_src_first, edgelist_dst_first, edge_value_first)); - thrust::for_each( - rmm::exec_policy(stream_view), - edge_first, - edge_first + number_of_edges, - update_edge_t{ - offset_view, index_view, get_dataframe_buffer_begin(values), major_range_first}); + auto edgelist_majors = std::move(store_transposed ? edgelist_dsts : edgelist_srcs); + auto edgelist_minors = std::move(store_transposed ? edgelist_srcs : edgelist_dsts); + + rmm::device_uvector offsets(0, stream_view); + rmm::device_uvector indices(0, stream_view); + auto values = allocate_dataframe_buffer(0, stream_view); + auto pair_first = thrust::make_zip_iterator(edgelist_majors.begin(), edgelist_minors.begin()); + if (edgelist_minors.size() > mem_frugal_threshold) { + offsets = compute_sparse_offsets(edgelist_majors.begin(), + edgelist_majors.end(), + major_range_first, + major_range_last, + false, + stream_view); + + auto pivot = major_range_first + static_cast(thrust::distance( + offsets.begin(), + thrust::lower_bound(rmm::exec_policy(stream_view), + offsets.begin(), + offsets.end(), + edgelist_minors.size() / 2))); + auto second_first = + detail::mem_frugal_partition(pair_first, + pair_first + edgelist_minors.size(), + get_dataframe_buffer_begin(edgelist_values), + thrust_tuple_get, 0>{}, + pivot, + stream_view); + thrust::sort_by_key(rmm::exec_policy(stream_view), + pair_first, + std::get<0>(second_first), + get_dataframe_buffer_begin(edgelist_values)); + thrust::sort_by_key(rmm::exec_policy(stream_view), + std::get<0>(second_first), + pair_first + edgelist_minors.size(), + std::get<1>(second_first)); + } else { + thrust::sort_by_key(rmm::exec_policy(stream_view), + pair_first, + pair_first + edgelist_minors.size(), + get_dataframe_buffer_begin(edgelist_values)); + + offsets = compute_sparse_offsets(edgelist_majors.begin(), + edgelist_majors.end(), + major_range_first, + major_range_last, + true, + stream_view); + } + indices = std::move(edgelist_minors); + values = std::move(edgelist_values); + + edgelist_majors.resize(0, stream_view); + edgelist_majors.shrink_to_fit(stream_view); std::optional> dcs_nzd_vertices{std::nullopt}; if (major_hypersparse_first) { @@ -226,47 +225,61 @@ compress_edgelist( } // compress edge list (COO) to CSR (or CSC) or CSR + DCSR (CSC + DCSC) hybrid -template -std::tuple< - rmm::device_uvector, - rmm::device_uvector::value_type>, - std::optional::value_type>>> -compress_edgelist( - VertexIterator edgelist_src_first, - VertexIterator edgelist_src_last, - VertexIterator edgelist_dst_first, - typename thrust::iterator_traits::value_type major_range_first, - std::optional::value_type> - major_hypersparse_first, - typename thrust::iterator_traits::value_type major_range_last, - typename thrust::iterator_traits::value_type /* minor_range_first */, - typename thrust::iterator_traits::value_type /* minor_range_last */, - rmm::cuda_stream_view stream_view) +template +std::tuple, + rmm::device_uvector, + std::optional>> +sort_and_compress_edgelist(rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + vertex_t major_range_first, + std::optional major_hypersparse_first, + vertex_t major_range_last, + vertex_t /* minor_range_first */, + vertex_t /* minor_range_last */, + size_t mem_frugal_threshold, + rmm::cuda_stream_view stream_view) { - using vertex_t = std::remove_cv_t::value_type>; - - auto number_of_edges = - static_cast(thrust::distance(edgelist_src_first, edgelist_src_last)); - - auto offsets = compute_sparse_offsets( - store_transposed ? edgelist_dst_first : edgelist_src_first, - store_transposed ? edgelist_dst_first + number_of_edges : edgelist_src_last, - major_range_first, - major_range_last, - stream_view); - - rmm::device_uvector indices(number_of_edges, stream_view); - thrust::fill(rmm::exec_policy(stream_view), indices.begin(), indices.end(), vertex_t{0}); - - auto offset_view = raft::device_span(offsets.data(), offsets.size()); - auto index_view = raft::device_span(indices.data(), indices.size()); - auto edge_first = - thrust::make_zip_iterator(thrust::make_tuple(edgelist_src_first, edgelist_dst_first)); - thrust::for_each(rmm::exec_policy(stream_view), - edge_first, - edge_first + number_of_edges, - update_edge_t{ - offset_view, index_view, static_cast(nullptr), major_range_first}); + auto edgelist_majors = std::move(store_transposed ? edgelist_dsts : edgelist_srcs); + auto edgelist_minors = std::move(store_transposed ? edgelist_srcs : edgelist_dsts); + + rmm::device_uvector offsets(0, stream_view); + rmm::device_uvector indices(0, stream_view); + auto edge_first = thrust::make_zip_iterator(edgelist_majors.begin(), edgelist_minors.begin()); + if (edgelist_minors.size() > mem_frugal_threshold) { + offsets = compute_sparse_offsets(edgelist_majors.begin(), + edgelist_majors.end(), + major_range_first, + major_range_last, + false, + stream_view); + + auto pivot = major_range_first + static_cast(thrust::distance( + offsets.begin(), + thrust::lower_bound(rmm::exec_policy(stream_view), + offsets.begin(), + offsets.end(), + edgelist_minors.size() / 2))); + auto second_first = + detail::mem_frugal_partition(edge_first, + edge_first + edgelist_minors.size(), + thrust_tuple_get, 0>{}, + pivot, + stream_view); + thrust::sort(rmm::exec_policy(stream_view), edge_first, second_first); + thrust::sort(rmm::exec_policy(stream_view), second_first, edge_first + edgelist_minors.size()); + } else { + thrust::sort(rmm::exec_policy(stream_view), edge_first, edge_first + edgelist_minors.size()); + offsets = compute_sparse_offsets(edgelist_majors.begin(), + edgelist_majors.end(), + major_range_first, + major_range_last, + true, + stream_view); + } + indices = std::move(edgelist_minors); + + edgelist_majors.resize(0, stream_view); + edgelist_majors.shrink_to_fit(stream_view); std::optional> dcs_nzd_vertices{std::nullopt}; if (major_hypersparse_first) { @@ -485,6 +498,63 @@ void sort_adjacency_list(raft::handle_t const& handle, } } -} // namespace detail +template +std::tuple> mark_entries(raft::handle_t const& handle, + size_t num_entries, + comparison_t comparison) +{ + rmm::device_uvector marked_entries(cugraph::packed_bool_size(num_entries), + handle.get_stream()); + + thrust::tabulate(handle.get_thrust_policy(), + marked_entries.begin(), + marked_entries.end(), + [comparison, num_entries] __device__(size_t idx) { + auto word = cugraph::packed_bool_empty_mask(); + size_t start_index = idx * cugraph::packed_bools_per_word(); + size_t bits_in_this_word = + (start_index + cugraph::packed_bools_per_word() < num_entries) + ? cugraph::packed_bools_per_word() + : (num_entries - start_index); + + for (size_t bit = 0; bit < bits_in_this_word; ++bit) { + if (comparison(start_index + bit)) word |= cugraph::packed_bool_mask(bit); + } + + return word; + }); + + size_t bit_count = thrust::transform_reduce( + handle.get_thrust_policy(), + marked_entries.begin(), + marked_entries.end(), + [] __device__(auto word) { return __popc(word); }, + size_t{0}, + thrust::plus()); + return std::make_tuple(bit_count, std::move(marked_entries)); +} + +template +rmm::device_uvector remove_flagged_elements(raft::handle_t const& handle, + rmm::device_uvector&& vector, + raft::device_span remove_flags, + size_t remove_count) +{ + rmm::device_uvector result(vector.size() - remove_count, handle.get_stream()); + + thrust::copy_if( + handle.get_thrust_policy(), + thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(vector.size()), + thrust::make_transform_output_iterator(result.begin(), + indirection_t{vector.data()}), + [remove_flags] __device__(size_t i) { + return !(remove_flags[cugraph::packed_bool_offset(i)] & cugraph::packed_bool_mask(i)); + }); + + return result; +} + +} // namespace detail } // namespace cugraph diff --git a/cpp/src/structure/graph_impl.cuh b/cpp/src/structure/graph_impl.cuh index 97975897e08..75862266789 100644 --- a/cpp/src/structure/graph_impl.cuh +++ b/cpp/src/structure/graph_impl.cuh @@ -213,16 +213,17 @@ update_local_sorted_unique_edge_majors_minors( thrust::make_counting_iterator(minor_range_first + num_scanned), thrust::make_counting_iterator(minor_range_first + num_scanned + this_scan_size), unique_edge_minors.begin() + num_copied, - cugraph::detail::check_bit_set_t{minor_bitmaps.data(), minor_range_first}))); + cugraph::detail::check_bit_set_t{minor_bitmaps.data(), + minor_range_first}))); num_scanned += this_scan_size; } #else - thrust::copy_if( - handle.get_thrust_policy(), - thrust::make_counting_iterator(minor_range_first), - thrust::make_counting_iterator(minor_range_last), - unique_edge_minors.begin(), - cugraph::detail::check_bit_set_t{minor_bitmaps.data(), minor_range_first}); + thrust::copy_if(handle.get_thrust_policy(), + thrust::make_counting_iterator(minor_range_first), + thrust::make_counting_iterator(minor_range_last), + unique_edge_minors.begin(), + cugraph::detail::check_bit_set_t{ + minor_bitmaps.data(), minor_range_first}); #endif auto num_chunks = diff --git a/cpp/src/structure/graph_view_impl.cuh b/cpp/src/structure/graph_view_impl.cuh index 7626784c13c..64a8a3212b3 100644 --- a/cpp/src/structure/graph_view_impl.cuh +++ b/cpp/src/structure/graph_view_impl.cuh @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -106,6 +107,7 @@ rmm::device_uvector compute_major_degrees( std::vector const& edge_partition_offsets, std::optional> const& edge_partition_dcs_nzd_vertices, std::optional> const& edge_partition_dcs_nzd_vertex_counts, + std::optional> const& edge_partition_masks, partition_t const& partition, std::vector const& edge_partition_segment_offsets) { @@ -142,7 +144,9 @@ rmm::device_uvector compute_major_degrees( vertex_t major_range_last{}; std::tie(major_range_first, major_range_last) = partition.vertex_partition_range(major_range_vertex_partition_id); - auto p_offsets = edge_partition_offsets[i]; + auto offsets = edge_partition_offsets[i]; + auto masks = + edge_partition_masks ? thrust::make_optional((*edge_partition_masks)[i]) : thrust::nullopt; auto segment_offset_size_per_partition = edge_partition_segment_offsets.size() / static_cast(minor_comm_size); auto major_hypersparse_first = @@ -155,9 +159,16 @@ rmm::device_uvector compute_major_degrees( thrust::make_counting_iterator(vertex_t{0}), thrust::make_counting_iterator(major_hypersparse_first - major_range_first), local_degrees.begin(), - [p_offsets] __device__(auto i) { return p_offsets[i + 1] - p_offsets[i]; }); + [offsets, masks] __device__(auto i) { + auto local_degree = offsets[i + 1] - offsets[i]; + if (masks) { + local_degree = static_cast( + detail::count_set_bits(*masks, offsets[i], local_degree)); + } + return local_degree; + }); if (use_dcs) { - auto p_dcs_nzd_vertices = (*edge_partition_dcs_nzd_vertices)[i]; + auto dcs_nzd_vertices = (*edge_partition_dcs_nzd_vertices)[i]; auto dcs_nzd_vertex_count = (*edge_partition_dcs_nzd_vertex_counts)[i]; thrust::fill(execution_policy, local_degrees.begin() + (major_hypersparse_first - major_range_first), @@ -166,15 +177,20 @@ rmm::device_uvector compute_major_degrees( thrust::for_each(execution_policy, thrust::make_counting_iterator(vertex_t{0}), thrust::make_counting_iterator(dcs_nzd_vertex_count), - [p_offsets, - p_dcs_nzd_vertices, + [offsets, + dcs_nzd_vertices, + masks, major_range_first, major_hypersparse_first, local_degrees = local_degrees.data()] __device__(auto i) { - auto d = p_offsets[(major_hypersparse_first - major_range_first) + i + 1] - - p_offsets[(major_hypersparse_first - major_range_first) + i]; - auto v = p_dcs_nzd_vertices[i]; - local_degrees[v - major_range_first] = d; + auto major_idx = (major_hypersparse_first - major_range_first) + i; + auto local_degree = offsets[major_idx + 1] - offsets[major_idx]; + if (masks) { + local_degree = static_cast( + detail::count_set_bits(*masks, offsets[major_idx], local_degree)); + } + auto v = dcs_nzd_vertices[i]; + local_degrees[v - major_range_first] = local_degree; }); } minor_comm.reduce(local_degrees.data(), @@ -193,12 +209,21 @@ rmm::device_uvector compute_major_degrees( template rmm::device_uvector compute_major_degrees(raft::handle_t const& handle, edge_t const* offsets, + std::optional masks, vertex_t number_of_vertices) { rmm::device_uvector degrees(number_of_vertices, handle.get_stream()); thrust::tabulate( - handle.get_thrust_policy(), degrees.begin(), degrees.end(), [offsets] __device__(auto i) { - return offsets[i + 1] - offsets[i]; + handle.get_thrust_policy(), + degrees.begin(), + degrees.end(), + [offsets, masks = masks ? thrust::make_optional(*masks) : thrust::nullopt] __device__(auto i) { + auto local_degree = offsets[i + 1] - offsets[i]; + if (masks) { + local_degree = + static_cast(detail::count_set_bits(*masks, offsets[i], local_degree)); + } + return local_degree; }); return degrees; } @@ -512,16 +537,18 @@ rmm::device_uvector graph_view_t>:: compute_in_degrees(raft::handle_t const& handle) const { - CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); - if (store_transposed) { return compute_major_degrees(handle, this->edge_partition_offsets_, this->edge_partition_dcs_nzd_vertices_, this->edge_partition_dcs_nzd_vertex_counts_, + this->has_edge_mask() + ? std::make_optional((*(this->edge_mask_view())).value_firsts()) + : std::nullopt, this->partition_, this->edge_partition_segment_offsets_); } else { + CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); return compute_minor_degrees(handle, *this); } } @@ -531,11 +558,15 @@ rmm::device_uvector graph_view_t>:: compute_in_degrees(raft::handle_t const& handle) const { - CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); - if (store_transposed) { - return compute_major_degrees(handle, this->offsets_, this->local_vertex_partition_range_size()); + return compute_major_degrees( + handle, + this->offsets_, + this->has_edge_mask() ? std::make_optional((*(this->edge_mask_view())).value_firsts()[0]) + : std::nullopt, + this->local_vertex_partition_range_size()); } else { + CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); return compute_minor_degrees(handle, *this); } } @@ -545,15 +576,17 @@ rmm::device_uvector graph_view_t>:: compute_out_degrees(raft::handle_t const& handle) const { - CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); - if (store_transposed) { + CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); return compute_minor_degrees(handle, *this); } else { return compute_major_degrees(handle, this->edge_partition_offsets_, this->edge_partition_dcs_nzd_vertices_, this->edge_partition_dcs_nzd_vertex_counts_, + this->has_edge_mask() + ? std::make_optional((*(this->edge_mask_view())).value_firsts()) + : std::nullopt, this->partition_, this->edge_partition_segment_offsets_); } @@ -564,12 +597,16 @@ rmm::device_uvector graph_view_t>:: compute_out_degrees(raft::handle_t const& handle) const { - CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); - if (store_transposed) { + CUGRAPH_EXPECTS(!has_edge_mask(), "unimplemented."); return compute_minor_degrees(handle, *this); } else { - return compute_major_degrees(handle, this->offsets_, this->local_vertex_partition_range_size()); + return compute_major_degrees( + handle, + this->offsets_, + this->has_edge_mask() ? std::make_optional((*(this->edge_mask_view())).value_firsts()[0]) + : std::nullopt, + this->local_vertex_partition_range_size()); } } diff --git a/cpp/src/structure/induced_subgraph_impl.cuh b/cpp/src/structure/induced_subgraph_impl.cuh index 950cca5828d..18e1af32a71 100644 --- a/cpp/src/structure/induced_subgraph_impl.cuh +++ b/cpp/src/structure/induced_subgraph_impl.cuh @@ -196,6 +196,7 @@ extract_induced_subgraphs( graph_ids_v.end(), size_t{0}, size_t{subgraph_offsets.size() - 1}, + true, handle.get_stream()); dst_subgraph_offsets = @@ -290,6 +291,7 @@ extract_induced_subgraphs( subgraph_edge_graph_ids.end(), size_t{0}, size_t{subgraph_offsets.size() - 1}, + true, handle.get_stream()); #ifdef TIMING diff --git a/cpp/src/structure/remove_multi_edges.cu b/cpp/src/structure/remove_multi_edges.cu new file mode 100644 index 00000000000..ba07d068c0e --- /dev/null +++ b/cpp/src/structure/remove_multi_edges.cu @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +namespace cugraph { + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +} // namespace cugraph diff --git a/cpp/src/structure/remove_multi_edges_impl.cuh b/cpp/src/structure/remove_multi_edges_impl.cuh new file mode 100644 index 00000000000..ab6b1fba8eb --- /dev/null +++ b/cpp/src/structure/remove_multi_edges_impl.cuh @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +// FIXME: mem_frugal_partition should probably not be in shuffle_comm.hpp +// It's used here without any notion of shuffling +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace cugraph { + +namespace detail { + +template +struct hash_src_dst_pair { + int32_t num_groups; + + int32_t __device__ operator()(thrust::tuple t) const + { + vertex_t pair[2]; + pair[0] = thrust::get<0>(t); + pair[1] = thrust::get<1>(t); + cuco::detail::MurmurHash3_32 hash_func{}; + return hash_func.compute_hash(reinterpret_cast(pair), 2 * sizeof(vertex_t)) % + num_groups; + } +}; + +template +std::tuple, rmm::device_uvector> group_multi_edges( + raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + size_t mem_frugal_threshold) +{ + auto pair_first = thrust::make_zip_iterator(edgelist_srcs.begin(), edgelist_dsts.begin()); + + if (edgelist_srcs.size() > mem_frugal_threshold) { + // FIXME: Tuning parameter to address high frequency multi-edges + // Defaulting to 2 which makes the code easier. If + // num_groups > 2 we can evaluate whether to find a good + // midpoint to do 2 sorts, or if we should do more than 2 sorts. + const size_t num_groups{2}; + + auto group_counts = groupby_and_count(pair_first, + pair_first + edgelist_srcs.size(), + hash_src_dst_pair{}, + num_groups, + mem_frugal_threshold, + handle.get_stream()); + + std::vector h_group_counts(group_counts.size()); + raft::update_host( + h_group_counts.data(), group_counts.data(), group_counts.size(), handle.get_stream()); + + thrust::sort(handle.get_thrust_policy(), pair_first, pair_first + h_group_counts[0]); + thrust::sort(handle.get_thrust_policy(), + pair_first + h_group_counts[0], + pair_first + edgelist_srcs.size()); + } else { + thrust::sort(handle.get_thrust_policy(), pair_first, pair_first + edgelist_srcs.size()); + } + + return std::make_tuple(std::move(edgelist_srcs), std::move(edgelist_dsts)); +} + +template +std::tuple, + rmm::device_uvector, + decltype(allocate_dataframe_buffer(size_t{0}, rmm::cuda_stream_view{}))> +group_multi_edges( + raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + decltype(allocate_dataframe_buffer(0, rmm::cuda_stream_view{}))&& edgelist_values, + size_t mem_frugal_threshold) +{ + auto pair_first = thrust::make_zip_iterator(edgelist_srcs.begin(), edgelist_dsts.begin()); + auto value_first = get_dataframe_buffer_begin(edgelist_values); + + if (edgelist_srcs.size() > mem_frugal_threshold) { + // FIXME: Tuning parameter to address high frequency multi-edges + // Defaulting to 2 which makes the code easier. If + // num_groups > 2 we can evaluate whether to find a good + // midpoint to do 2 sorts, or if we should do more than 2 sorts. + const size_t num_groups{2}; + + auto group_counts = groupby_and_count(pair_first, + pair_first + edgelist_srcs.size(), + value_first, + hash_src_dst_pair{}, + num_groups, + mem_frugal_threshold, + handle.get_stream()); + + std::vector h_group_counts(group_counts.size()); + raft::update_host( + h_group_counts.data(), group_counts.data(), group_counts.size(), handle.get_stream()); + + thrust::sort_by_key(handle.get_thrust_policy(), + pair_first, + pair_first + h_group_counts[0], + get_dataframe_buffer_begin(edgelist_values)); + thrust::sort_by_key(handle.get_thrust_policy(), + pair_first + h_group_counts[0], + pair_first + edgelist_srcs.size(), + get_dataframe_buffer_begin(edgelist_values) + h_group_counts[0]); + } else { + thrust::sort_by_key(handle.get_thrust_policy(), + pair_first, + pair_first + edgelist_srcs.size(), + get_dataframe_buffer_begin(edgelist_values)); + } + + return std::make_tuple( + std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_values)); +} + +} // namespace detail + +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types) +{ + auto total_global_mem = handle.get_device_properties().totalGlobalMem; + size_t element_size = sizeof(vertex_t) * 2; + if (edgelist_weights) { element_size += sizeof(weight_t); } + if (edgelist_edge_ids) { element_size += sizeof(edge_t); } + if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } + + auto constexpr mem_frugal_ratio = + 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the + // total_global_mem, switch to the memory frugal approach + auto mem_frugal_threshold = + static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); + + if (edgelist_weights) { + if (edgelist_edge_ids) { + if (edgelist_edge_types) { + std::forward_as_tuple(edgelist_srcs, + edgelist_dsts, + std::tie(edgelist_weights, edgelist_edge_ids, edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), + std::move(*edgelist_edge_ids), + std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::forward_as_tuple( + edgelist_srcs, edgelist_dsts, std::tie(edgelist_weights, edgelist_edge_ids)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), std::move(*edgelist_edge_ids)), + mem_frugal_threshold); + } + } else { + if (edgelist_edge_types) { + std::forward_as_tuple( + edgelist_srcs, edgelist_dsts, std::tie(edgelist_weights, edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::forward_as_tuple(edgelist_srcs, edgelist_dsts, std::tie(edgelist_weights)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights)), + mem_frugal_threshold); + } + } + } else { + if (edgelist_edge_ids) { + if (edgelist_edge_types) { + std::forward_as_tuple( + edgelist_srcs, edgelist_dsts, std::tie(edgelist_edge_ids, edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_ids), std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::forward_as_tuple(edgelist_srcs, edgelist_dsts, std::tie(edgelist_edge_ids)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_ids)), + mem_frugal_threshold); + } + } else { + if (edgelist_edge_types) { + std::forward_as_tuple(edgelist_srcs, edgelist_dsts, std::tie(edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::tie(edgelist_srcs, edgelist_dsts) = detail::group_multi_edges( + handle, std::move(edgelist_srcs), std::move(edgelist_dsts), mem_frugal_threshold); + } + } + } + + auto [multi_edge_count, multi_edges_to_delete] = + detail::mark_entries(handle, + edgelist_srcs.size(), + [d_edgelist_srcs = edgelist_srcs.data(), + d_edgelist_dsts = edgelist_dsts.data()] __device__(auto idx) { + return (idx > 0) && (d_edgelist_srcs[idx - 1] == d_edgelist_srcs[idx]) && + (d_edgelist_dsts[idx - 1] == d_edgelist_dsts[idx]); + }); + + if (multi_edge_count > 0) { + edgelist_srcs = detail::remove_flagged_elements( + handle, + std::move(edgelist_srcs), + raft::device_span{multi_edges_to_delete.data(), multi_edges_to_delete.size()}, + multi_edge_count); + edgelist_dsts = detail::remove_flagged_elements( + handle, + std::move(edgelist_dsts), + raft::device_span{multi_edges_to_delete.data(), multi_edges_to_delete.size()}, + multi_edge_count); + + if (edgelist_weights) + edgelist_weights = detail::remove_flagged_elements( + handle, + std::move(*edgelist_weights), + raft::device_span{multi_edges_to_delete.data(), + multi_edges_to_delete.size()}, + multi_edge_count); + + if (edgelist_edge_ids) + edgelist_edge_ids = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_ids), + raft::device_span{multi_edges_to_delete.data(), + multi_edges_to_delete.size()}, + multi_edge_count); + + if (edgelist_edge_types) + edgelist_edge_types = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_types), + raft::device_span{multi_edges_to_delete.data(), + multi_edges_to_delete.size()}, + multi_edge_count); + } + + return std::make_tuple(std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); +} + +} // namespace cugraph diff --git a/cpp/src/structure/remove_self_loops.cu b/cpp/src/structure/remove_self_loops.cu new file mode 100644 index 00000000000..8a66c1e05e3 --- /dev/null +++ b/cpp/src/structure/remove_self_loops.cu @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +namespace cugraph { + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +} // namespace cugraph diff --git a/cpp/src/structure/remove_self_loops_impl.cuh b/cpp/src/structure/remove_self_loops_impl.cuh new file mode 100644 index 00000000000..161ffeae28e --- /dev/null +++ b/cpp/src/structure/remove_self_loops_impl.cuh @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace cugraph { + +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types) +{ + auto [self_loop_count, self_loops_to_delete] = + detail::mark_entries(handle, + edgelist_srcs.size(), + [d_srcs = edgelist_srcs.data(), d_dsts = edgelist_dsts.data()] __device__( + size_t i) { return d_srcs[i] == d_dsts[i]; }); + + if (self_loop_count > 0) { + edgelist_srcs = detail::remove_flagged_elements( + handle, + std::move(edgelist_srcs), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + edgelist_dsts = detail::remove_flagged_elements( + handle, + std::move(edgelist_dsts), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + + if (edgelist_weights) + edgelist_weights = detail::remove_flagged_elements( + handle, + std::move(*edgelist_weights), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + + if (edgelist_edge_ids) + edgelist_edge_ids = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_ids), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + + if (edgelist_edge_types) + edgelist_edge_types = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_types), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + } + + return std::make_tuple(std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); +} + +} // namespace cugraph diff --git a/cpp/src/structure/renumber_edgelist_impl.cuh b/cpp/src/structure/renumber_edgelist_impl.cuh index 6bc19ff4fe1..09a4dae6c64 100644 --- a/cpp/src/structure/renumber_edgelist_impl.cuh +++ b/cpp/src/structure/renumber_edgelist_impl.cuh @@ -367,18 +367,19 @@ std::tuple, std::vector, vertex_t> compu rmm::device_uvector sorted_local_vertex_degrees(0, handle.get_stream()); std::optional> stream_pool_indices{ std::nullopt}; // FIXME: move this inside the if statement + + auto constexpr num_chunks = size_t{ + 2}; // tuning parameter, this trade-offs # binary searches (up to num_chunks times more binary + // searches can be necessary if num_unique_majors << edgelist_edge_counts[i]) and temporary + // buffer requirement (cut by num_chunks times), currently set to 2 to avoid peak memory + // usage happening in this part (especially when minor_comm_size is small) + if constexpr (multi_gpu) { auto& comm = handle.get_comms(); auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); auto const minor_comm_rank = minor_comm.get_rank(); auto const minor_comm_size = minor_comm.get_size(); - auto constexpr num_chunks = size_t{ - 2}; // tuning parameter, this trade-offs # binary searches (up to num_chunks times more - // binary searches can be necessary if num_unique_majors << edgelist_edge_counts[i]) and - // temporary buffer requirement (cut by num_chunks times), currently set to 2 to avoid - // peak memory usage happening in this part (especially when minor_comm_size is small) - assert(edgelist_majors.size() == minor_comm_size); auto edge_partition_major_range_sizes = @@ -433,29 +434,30 @@ std::tuple, std::vector, vertex_t> compu sorted_major_degrees.end(), edge_t{0}); - rmm::device_uvector tmp_majors( + rmm::device_uvector tmp_majors(0, loop_stream); + tmp_majors.reserve( (static_cast(edgelist_edge_counts[i]) + (num_chunks - 1)) / num_chunks, - handle.get_stream()); + loop_stream); size_t offset{0}; for (size_t j = 0; j < num_chunks; ++j) { size_t this_chunk_size = - std::min(tmp_majors.size(), static_cast(edgelist_edge_counts[i]) - offset); + std::min(tmp_majors.capacity(), static_cast(edgelist_edge_counts[i]) - offset); + tmp_majors.resize(this_chunk_size, loop_stream); thrust::copy(rmm::exec_policy(loop_stream), edgelist_majors[i] + offset, - edgelist_majors[i] + offset + this_chunk_size, + edgelist_majors[i] + offset + tmp_majors.size(), tmp_majors.begin()); - thrust::sort( - rmm::exec_policy(loop_stream), tmp_majors.begin(), tmp_majors.begin() + this_chunk_size); + thrust::sort(rmm::exec_policy(loop_stream), tmp_majors.begin(), tmp_majors.end()); auto num_unique_majors = thrust::count_if(rmm::exec_policy(loop_stream), thrust::make_counting_iterator(size_t{0}), - thrust::make_counting_iterator(this_chunk_size), + thrust::make_counting_iterator(tmp_majors.size()), is_first_in_run_t{tmp_majors.data()}); rmm::device_uvector tmp_keys(num_unique_majors, loop_stream); rmm::device_uvector tmp_values(num_unique_majors, loop_stream); thrust::reduce_by_key(rmm::exec_policy(loop_stream), tmp_majors.begin(), - tmp_majors.begin() + this_chunk_size, + tmp_majors.end(), thrust::make_constant_iterator(edge_t{1}), tmp_keys.begin(), tmp_values.begin()); @@ -486,44 +488,50 @@ std::tuple, std::vector, vertex_t> compu } else { assert(edgelist_majors.size() == 1); - rmm::device_uvector tmp_majors(edgelist_edge_counts[0], handle.get_stream()); - thrust::copy(handle.get_thrust_policy(), - edgelist_majors[0], - edgelist_majors[0] + edgelist_edge_counts[0], - tmp_majors.begin()); - thrust::sort(handle.get_thrust_policy(), tmp_majors.begin(), tmp_majors.end()); - auto num_unique_majors = - thrust::count_if(handle.get_thrust_policy(), - thrust::make_counting_iterator(size_t{0}), - thrust::make_counting_iterator(tmp_majors.size()), - is_first_in_run_t{tmp_majors.data()}); - rmm::device_uvector tmp_keys(num_unique_majors, handle.get_stream()); - rmm::device_uvector tmp_values(num_unique_majors, handle.get_stream()); - thrust::reduce_by_key(handle.get_thrust_policy(), - tmp_majors.begin(), - tmp_majors.end(), - thrust::make_constant_iterator(edge_t{1}), - tmp_keys.begin(), - tmp_values.begin()); - - tmp_majors.resize(0, handle.get_stream()); - tmp_majors.shrink_to_fit(handle.get_stream()); - sorted_local_vertex_degrees.resize(sorted_local_vertices.size(), handle.get_stream()); thrust::fill(handle.get_thrust_policy(), sorted_local_vertex_degrees.begin(), sorted_local_vertex_degrees.end(), edge_t{0}); - auto kv_pair_first = - thrust::make_zip_iterator(thrust::make_tuple(tmp_keys.begin(), tmp_values.begin())); - thrust::for_each(handle.get_thrust_policy(), - kv_pair_first, - kv_pair_first + tmp_keys.size(), - search_and_increment_degree_t{ - sorted_local_vertices.data(), - static_cast(sorted_local_vertices.size()), - sorted_local_vertex_degrees.data()}); + rmm::device_uvector tmp_majors(0, handle.get_stream()); + tmp_majors.reserve(static_cast(edgelist_edge_counts[0] + (num_chunks - 1)) / num_chunks, + handle.get_stream()); + size_t offset{0}; + for (size_t i = 0; i < num_chunks; ++i) { + size_t this_chunk_size = + std::min(tmp_majors.capacity(), static_cast(edgelist_edge_counts[0]) - offset); + tmp_majors.resize(this_chunk_size, handle.get_stream()); + thrust::copy(handle.get_thrust_policy(), + edgelist_majors[0] + offset, + edgelist_majors[0] + offset + tmp_majors.size(), + tmp_majors.begin()); + thrust::sort(handle.get_thrust_policy(), tmp_majors.begin(), tmp_majors.end()); + auto num_unique_majors = + thrust::count_if(handle.get_thrust_policy(), + thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(tmp_majors.size()), + is_first_in_run_t{tmp_majors.data()}); + rmm::device_uvector tmp_keys(num_unique_majors, handle.get_stream()); + rmm::device_uvector tmp_values(num_unique_majors, handle.get_stream()); + thrust::reduce_by_key(handle.get_thrust_policy(), + tmp_majors.begin(), + tmp_majors.end(), + thrust::make_constant_iterator(edge_t{1}), + tmp_keys.begin(), + tmp_values.begin()); + + auto kv_pair_first = + thrust::make_zip_iterator(thrust::make_tuple(tmp_keys.begin(), tmp_values.begin())); + thrust::for_each(handle.get_thrust_policy(), + kv_pair_first, + kv_pair_first + tmp_keys.size(), + search_and_increment_degree_t{ + sorted_local_vertices.data(), + static_cast(sorted_local_vertices.size()), + sorted_local_vertex_degrees.data()}); + offset += this_chunk_size; + } } // 4. sort local vertices by degree (descending) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 2a4bb8ab2a5..6530a25d178 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -288,10 +288,6 @@ ConfigureTest(GENERATE_RMAT_TEST generators/generate_rmat_test.cpp) ConfigureTest(GENERATE_BIPARTITE_RMAT_TEST generators/generate_bipartite_rmat_test.cpp GPUS 1 PERCENT 100) -################################################################################################### -# - Graph mask tests ------------------------------------------------------------------------------ -ConfigureTest(GRAPH_MASK_TEST structure/graph_mask_test.cpp) - ################################################################################################### # - Symmetrize tests ------------------------------------------------------------------------------ ConfigureTest(SYMMETRIZE_TEST structure/symmetrize_test.cpp) @@ -419,13 +415,6 @@ ConfigureTest(K_HOP_NBRS_TEST traversal/k_hop_nbrs_test.cpp) # - install tests --------------------------------------------------------------------------------- rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcugraph) -################################################################################################### -# - MTMG tests ------------------------------------------------------------------------- -ConfigureTest(MTMG_TEST mtmg/threaded_test.cu) -target_link_libraries(MTMG_TEST - PRIVATE - UCP::UCP - ) ################################################################################################### # - MG tests -------------------------------------------------------------------------------------- @@ -684,6 +673,7 @@ if(BUILD_CUGRAPH_MG_TESTS) ConfigureCTestMG(MG_CAPI_TWO_HOP_NEIGHBORS_TEST c_api/mg_two_hop_neighbors_test.c) rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_mg DESTINATION bin/gtests/libcugraph_mg) + endif() ################################################################################################### @@ -743,4 +733,25 @@ ConfigureCTest(CAPI_EGONET_TEST c_api/egonet_test.c) ConfigureCTest(CAPI_TWO_HOP_NEIGHBORS_TEST c_api/two_hop_neighbors_test.c) ConfigureCTest(CAPI_LEGACY_K_TRUSS_TEST c_api/legacy_k_truss_test.c) +if (BUILD_CUGRAPH_MTMG_TESTS) + ################################################################################################### + # - MTMG tests ------------------------------------------------------------------------- + ConfigureTest(MTMG_TEST mtmg/threaded_test.cu) + target_link_libraries(MTMG_TEST + PRIVATE + UCP::UCP + ) + + if(BUILD_CUGRAPH_MG_TESTS) + ############################################################################################### + # - Multi-node MTMG tests --------------------------------------------------------------------- + ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp) + target_link_libraries(MTMG_MULTINODE_TEST + PRIVATE + cugraphmgtestutil + UCP::UCP + ) + endif(BUILD_CUGRAPH_MG_TESTS) +endif(BUILD_CUGRAPH_MTMG_TESTS) + rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_c DESTINATION bin/gtests/libcugraph_c) diff --git a/cpp/tests/c_api/create_graph_test.c b/cpp/tests/c_api/create_graph_test.c index 736db761ebd..11da2eb8589 100644 --- a/cpp/tests/c_api/create_graph_test.c +++ b/cpp/tests/c_api/create_graph_test.c @@ -91,8 +91,9 @@ int test_create_sg_graph_simple() handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_sg_graph_create(handle, + ret_code = cugraph_graph_create_sg(handle, &properties, + NULL, src_view, dst_view, wgt_view, @@ -101,11 +102,13 @@ int test_create_sg_graph_simple() FALSE, FALSE, FALSE, + FALSE, + FALSE, &graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); - cugraph_sg_graph_free(graph); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(dst_view); @@ -300,7 +303,7 @@ int test_create_sg_graph_csr() } cugraph_sample_result_free(result); - cugraph_sg_graph_free(graph); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(indices_view); cugraph_type_erased_device_array_view_free(offsets_view); @@ -382,8 +385,9 @@ int test_create_sg_graph_symmetric_error() handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_sg_graph_create(handle, + ret_code = cugraph_graph_create_sg(handle, &properties, + NULL, src_view, dst_view, wgt_view, @@ -391,19 +395,500 @@ int test_create_sg_graph_symmetric_error() NULL, FALSE, FALSE, + FALSE, + FALSE, TRUE, &graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code != CUGRAPH_SUCCESS, "graph creation succeeded but should have failed."); - if (ret_code == CUGRAPH_SUCCESS) cugraph_sg_graph_free(graph); + if (ret_code == CUGRAPH_SUCCESS) cugraph_graph_free(graph); + + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(dst_view); + cugraph_type_erased_device_array_view_free(src_view); + cugraph_type_erased_device_array_free(wgt); + cugraph_type_erased_device_array_free(dst); + cugraph_type_erased_device_array_free(src); + + cugraph_free_resource_handle(handle); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_sg_graph_with_isolated_vertices() +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 8; + size_t num_vertices = 7; + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_resource_handle_t* handle = NULL; + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + handle = cugraph_create_resource_handle(NULL); + TEST_ASSERT(test_ret_value, handle != NULL, "resource handle creation failed."); + + cugraph_type_erased_device_array_t* vertices; + cugraph_type_erased_device_array_t* src; + cugraph_type_erased_device_array_t* dst; + cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* vertices_view; + cugraph_type_erased_device_array_view_t* src_view; + cugraph_type_erased_device_array_view_t* dst_view; + cugraph_type_erased_device_array_view_t* wgt_view; + + ret_code = + cugraph_type_erased_device_array_create(handle, num_vertices, vertex_tid, &vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view = cugraph_type_erased_device_array_view(vertices); + src_view = cugraph_type_erased_device_array_view(src); + dst_view = cugraph_type_erased_device_array_view(dst); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view, (byte_t*)h_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view, (byte_t*)h_src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view, (byte_t*)h_dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view, (byte_t*)h_wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + + ret_code = cugraph_graph_create_sg(handle, + &properties, + vertices_view, + src_view, + dst_view, + wgt_view, + NULL, + NULL, + FALSE, + FALSE, + FALSE, + FALSE, + FALSE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_centrality_result_t* result = NULL; + + // To verify we will call pagerank + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + vertex_t h_result_vertices[num_vertices]; + weight_t h_pageranks[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(dst_view); + cugraph_type_erased_device_array_view_free(src_view); + cugraph_type_erased_device_array_view_free(vertices_view); + cugraph_type_erased_device_array_free(wgt); + cugraph_type_erased_device_array_free(dst); + cugraph_type_erased_device_array_free(src); + cugraph_type_erased_device_array_free(vertices); + + cugraph_free_resource_handle(handle); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_sg_graph_csr_with_isolated() +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 8; + size_t num_vertices = 7; + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + /* + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + */ + edge_t h_offsets[] = {0, 1, 3, 6, 7, 8, 8, 8}; + vertex_t h_indices[] = {1, 3, 4, 0, 1, 3, 5, 5}; + vertex_t h_start[] = {0, 1, 2, 3, 4, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_resource_handle_t* handle = NULL; + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + handle = cugraph_create_resource_handle(NULL); + TEST_ASSERT(test_ret_value, handle != NULL, "resource handle creation failed."); + + cugraph_type_erased_device_array_t* offsets; + cugraph_type_erased_device_array_t* indices; + cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* offsets_view; + cugraph_type_erased_device_array_view_t* indices_view; + cugraph_type_erased_device_array_view_t* wgt_view; + + ret_code = cugraph_type_erased_device_array_create( + handle, num_vertices + 1, vertex_tid, &offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "offsets create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &indices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "indices create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + offsets_view = cugraph_type_erased_device_array_view(offsets); + indices_view = cugraph_type_erased_device_array_view(indices); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, offsets_view, (byte_t*)h_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "offsets copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, indices_view, (byte_t*)h_indices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "indices copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view, (byte_t*)h_wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + + ret_code = cugraph_sg_graph_create_from_csr(handle, + &properties, + offsets_view, + indices_view, + wgt_view, + NULL, + NULL, + FALSE, + FALSE, + FALSE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_centrality_result_t* result = NULL; + + // To verify we will call pagerank + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + vertex_t h_result_vertices[num_vertices]; + weight_t h_pageranks[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(indices_view); + cugraph_type_erased_device_array_view_free(offsets_view); + cugraph_type_erased_device_array_free(wgt); + cugraph_type_erased_device_array_free(indices); + cugraph_type_erased_device_array_free(offsets); + + cugraph_free_resource_handle(handle); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_sg_graph_with_isolated_vertices_multi_input() +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 66; + size_t num_vertices = 7; + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_resource_handle_t* handle = NULL; + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + handle = cugraph_create_resource_handle(NULL); + TEST_ASSERT(test_ret_value, handle != NULL, "resource handle creation failed."); + + cugraph_type_erased_device_array_t* vertices; + cugraph_type_erased_device_array_t* src; + cugraph_type_erased_device_array_t* dst; + cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* vertices_view; + cugraph_type_erased_device_array_view_t* src_view; + cugraph_type_erased_device_array_view_t* dst_view; + cugraph_type_erased_device_array_view_t* wgt_view; + + ret_code = + cugraph_type_erased_device_array_create(handle, num_vertices, vertex_tid, &vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view = cugraph_type_erased_device_array_view(vertices); + src_view = cugraph_type_erased_device_array_view(src); + dst_view = cugraph_type_erased_device_array_view(dst); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view, (byte_t*)h_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view, (byte_t*)h_src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view, (byte_t*)h_dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view, (byte_t*)h_wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + + ret_code = cugraph_graph_create_sg(handle, + &properties, + vertices_view, + src_view, + dst_view, + wgt_view, + NULL, + NULL, + FALSE, + FALSE, + TRUE, + TRUE, + FALSE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_centrality_result_t* result = NULL; + + // To verify we will call pagerank + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + vertex_t h_result_vertices[num_vertices]; + weight_t h_pageranks[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(dst_view); cugraph_type_erased_device_array_view_free(src_view); + cugraph_type_erased_device_array_view_free(vertices_view); cugraph_type_erased_device_array_free(wgt); cugraph_type_erased_device_array_free(dst); cugraph_type_erased_device_array_free(src); + cugraph_type_erased_device_array_free(vertices); cugraph_free_resource_handle(handle); cugraph_error_free(ret_error); @@ -419,5 +904,8 @@ int main(int argc, char** argv) result |= RUN_TEST(test_create_sg_graph_simple); result |= RUN_TEST(test_create_sg_graph_csr); result |= RUN_TEST(test_create_sg_graph_symmetric_error); + result |= RUN_TEST(test_create_sg_graph_with_isolated_vertices); + result |= RUN_TEST(test_create_sg_graph_csr_with_isolated); + result |= RUN_TEST(test_create_sg_graph_with_isolated_vertices_multi_input); return result; } diff --git a/cpp/tests/c_api/eigenvector_centrality_test.c b/cpp/tests/c_api/eigenvector_centrality_test.c index 9fd2d2bee6f..8bc5971a70c 100644 --- a/cpp/tests/c_api/eigenvector_centrality_test.c +++ b/cpp/tests/c_api/eigenvector_centrality_test.c @@ -109,11 +109,30 @@ int test_eigenvector_centrality() h_src, h_dst, h_wgt, h_result, num_vertices, num_edges, TRUE, epsilon, max_iterations); } +int test_eigenvector_centrality_3971() +{ + size_t num_edges = 4; + size_t num_vertices = 3; + + vertex_t h_src[] = {0, 1, 1, 2}; + vertex_t h_dst[] = {1, 0, 2, 1}; + weight_t h_wgt[] = {1.0f, 1.0f, 1.0f, 1.0f}; + weight_t h_result[] = {0.5, 0.707107, 0.5}; + + double epsilon = 1e-6; + size_t max_iterations = 1000; + + // Eigenvector centrality wants store_transposed = TRUE + return generic_eigenvector_centrality_test( + h_src, h_dst, h_wgt, h_result, num_vertices, num_edges, TRUE, epsilon, max_iterations); +} + /******************************************************************************/ int main(int argc, char** argv) { int result = 0; result |= RUN_TEST(test_eigenvector_centrality); + result |= RUN_TEST(test_eigenvector_centrality_3971); return result; } diff --git a/cpp/tests/c_api/leiden_test.c b/cpp/tests/c_api/leiden_test.c index 9e91adf9f89..df206ebd1ed 100644 --- a/cpp/tests/c_api/leiden_test.c +++ b/cpp/tests/c_api/leiden_test.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -161,7 +161,7 @@ int test_leiden_no_weights() vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 1, 3, 4, 0, 1, 3, 5, 5}; vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 0, 1, 1, 2, 2, 2, 3, 4}; vertex_t h_result[] = {1, 1, 1, 2, 0, 0}; - weight_t expected_modularity = 0.0859375; + weight_t expected_modularity = 0.125; // Louvain wants store_transposed = FALSE return generic_leiden_test(h_src, diff --git a/cpp/tests/c_api/louvain_test.c b/cpp/tests/c_api/louvain_test.c index e9ac5c9ff06..41d777545b2 100644 --- a/cpp/tests/c_api/louvain_test.c +++ b/cpp/tests/c_api/louvain_test.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,22 +46,39 @@ int generic_louvain_test(vertex_t* h_src, cugraph_graph_t* p_graph = NULL; cugraph_hierarchical_clustering_result_t* p_result = NULL; - data_type_id_t vertex_tid = INT32; - data_type_id_t edge_tid = INT32; - data_type_id_t weight_tid = FLOAT32; + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; data_type_id_t edge_id_tid = INT32; data_type_id_t edge_type_tid = INT32; p_handle = cugraph_create_resource_handle(NULL); TEST_ASSERT(test_ret_value, p_handle != NULL, "resource handle creation failed."); - ret_code = create_sg_test_graph(p_handle, vertex_tid, edge_tid, h_src, h_dst, weight_tid, h_wgt, edge_type_tid, NULL, edge_id_tid, NULL, num_edges, store_transposed, FALSE, FALSE, FALSE, &p_graph, &ret_error); + ret_code = create_sg_test_graph(p_handle, + vertex_tid, + edge_tid, + h_src, + h_dst, + weight_tid, + h_wgt, + edge_type_tid, + NULL, + edge_id_tid, + NULL, + num_edges, + store_transposed, + FALSE, + FALSE, + FALSE, + &p_graph, + &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "create_test_graph failed."); TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); - ret_code = - cugraph_louvain(p_handle, p_graph, max_level, threshold, resolution, FALSE, &p_result, &ret_error); + ret_code = cugraph_louvain( + p_handle, p_graph, max_level, threshold, resolution, FALSE, &p_result, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, "cugraph_louvain failed."); @@ -141,10 +158,10 @@ int test_louvain_no_weight() weight_t threshold = 1e-7; weight_t resolution = 1.0; - vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 1, 3, 4, 0, 1, 3, 5, 5}; - vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 0, 1, 1, 2, 2, 2, 3, 4}; - vertex_t h_result[] = {1, 1, 1, 2, 0, 0}; - weight_t expected_modularity = 0.0859375; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 1, 3, 4, 0, 1, 3, 5, 5}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_result[] = {1, 1, 1, 1, 0, 0}; + weight_t expected_modularity = 0.125; // Louvain wants store_transposed = FALSE return generic_louvain_test(h_src, diff --git a/cpp/tests/c_api/mg_create_graph_test.c b/cpp/tests/c_api/mg_create_graph_test.c index 4c8f2f22982..fec319d1881 100644 --- a/cpp/tests/c_api/mg_create_graph_test.c +++ b/cpp/tests/c_api/mg_create_graph_test.c @@ -17,6 +17,8 @@ #include "c_test_utils.h" /* RUN_TEST */ #include "mg_test_utils.h" /* RUN_TEST */ +#include + #include #include #include @@ -41,7 +43,7 @@ int test_create_mg_graph_simple(const cugraph_resource_handle_t* handle) vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; - cugraph_graph_t* p_graph = NULL; + cugraph_graph_t* graph = NULL; cugraph_graph_properties_t properties; properties.is_symmetric = FALSE; @@ -94,21 +96,25 @@ int test_create_mg_graph_simple(const cugraph_resource_handle_t* handle) handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_mg_graph_create(handle, + ret_code = cugraph_graph_create_mg(handle, &properties, - src_view, - dst_view, - wgt_view, + NULL, + (cugraph_type_erased_device_array_view_t const* const*) &src_view, + (cugraph_type_erased_device_array_view_t const* const*) &dst_view, + (cugraph_type_erased_device_array_view_t const* const*) &wgt_view, NULL, NULL, FALSE, - num_edges, + 1, + FALSE, + FALSE, TRUE, - &p_graph, + &graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); - cugraph_mg_graph_free(p_graph); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(dst_view); @@ -122,6 +128,382 @@ int test_create_mg_graph_simple(const cugraph_resource_handle_t* handle) return test_ret_value; } +int test_create_mg_graph_multiple_edge_lists(const cugraph_resource_handle_t* handle) +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 8; + size_t num_vertices = 7; + + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + const size_t num_local_arrays = 2; + + cugraph_type_erased_device_array_t* vertices[num_local_arrays]; + cugraph_type_erased_device_array_t* src[num_local_arrays]; + cugraph_type_erased_device_array_t* dst[num_local_arrays]; + cugraph_type_erased_device_array_t* wgt[num_local_arrays]; + cugraph_type_erased_device_array_view_t* vertices_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* src_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* dst_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* wgt_view[num_local_arrays]; + + int my_rank = cugraph_resource_handle_get_rank(handle); + int comm_size = cugraph_resource_handle_get_comm_size(handle); + + size_t local_num_vertices = (num_vertices + comm_size - 1) / comm_size; + size_t local_start_vertex = my_rank * local_num_vertices; + size_t local_num_edges = (num_edges + comm_size - 1) / comm_size; + size_t local_start_edge = my_rank * local_num_edges; + + local_num_edges = (local_num_edges < (num_edges - local_start_edge)) ? local_num_edges : (num_edges - local_start_edge); + local_num_vertices = (local_num_vertices < (num_vertices - local_start_vertex)) ? local_num_vertices : (num_vertices - local_start_vertex); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + size_t vertex_count = (local_num_vertices + num_local_arrays - 1) / num_local_arrays; + size_t vertex_start = i * vertex_count; + vertex_count = (vertex_count < (local_num_vertices - vertex_start)) ? vertex_count : (local_num_vertices - vertex_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, vertex_count, vertex_tid, vertices + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + size_t edge_count = (local_num_edges + num_local_arrays - 1) / num_local_arrays; + size_t edge_start = i * edge_count; + edge_count = (edge_count < (local_num_edges - edge_start)) ? edge_count : (local_num_edges - edge_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, src + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, dst + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, weight_tid, wgt + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view[i] = cugraph_type_erased_device_array_view(vertices[i]); + src_view[i] = cugraph_type_erased_device_array_view(src[i]); + dst_view[i] = cugraph_type_erased_device_array_view(dst[i]); + wgt_view[i] = cugraph_type_erased_device_array_view(wgt[i]); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view[i], (byte_t*)(h_vertices + local_start_vertex + vertex_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view[i], (byte_t*)(h_src + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view[i], (byte_t*)(h_dst + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view[i], (byte_t*)(h_wgt + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + } + + ret_code = cugraph_graph_create_mg(handle, + &properties, + (cugraph_type_erased_device_array_view_t const* const*) vertices_view, + (cugraph_type_erased_device_array_view_t const* const*) src_view, + (cugraph_type_erased_device_array_view_t const* const*) dst_view, + (cugraph_type_erased_device_array_view_t const* const*) wgt_view, + NULL, + NULL, + FALSE, + num_local_arrays, + FALSE, + FALSE, + TRUE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + // + // Now call pagerank and check results... + // + cugraph_centrality_result_t* result = NULL; + + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + + // NOTE: Because we get back vertex ids and pageranks, we can simply compare + // the returned values with the expected results for the entire + // graph. Each GPU will have a subset of the total vertices, so + // they will do a subset of the comparisons. + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(result_vertices); + + vertex_t h_result_vertices[num_local_vertices]; + weight_t h_pageranks[num_local_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + cugraph_type_erased_device_array_view_free(wgt_view[i]); + cugraph_type_erased_device_array_view_free(dst_view[i]); + cugraph_type_erased_device_array_view_free(src_view[i]); + cugraph_type_erased_device_array_view_free(vertices_view[i]); + cugraph_type_erased_device_array_free(wgt[i]); + cugraph_type_erased_device_array_free(dst[i]); + cugraph_type_erased_device_array_free(src[i]); + cugraph_type_erased_device_array_free(vertices[i]); + } + + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_mg_graph_multiple_edge_lists_multi_edge(const cugraph_resource_handle_t* handle) +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 11; + size_t num_vertices = 7; + + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.1f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + const size_t num_local_arrays = 2; + + cugraph_type_erased_device_array_t* vertices[num_local_arrays]; + cugraph_type_erased_device_array_t* src[num_local_arrays]; + cugraph_type_erased_device_array_t* dst[num_local_arrays]; + cugraph_type_erased_device_array_t* wgt[num_local_arrays]; + cugraph_type_erased_device_array_view_t* vertices_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* src_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* dst_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* wgt_view[num_local_arrays]; + + int my_rank = cugraph_resource_handle_get_rank(handle); + int comm_size = cugraph_resource_handle_get_comm_size(handle); + + size_t local_num_vertices = (num_vertices + comm_size - 1) / comm_size; + size_t local_start_vertex = my_rank * local_num_vertices; + size_t local_num_edges = (num_edges + comm_size - 1) / comm_size; + size_t local_start_edge = my_rank * local_num_edges; + + local_num_edges = (local_num_edges < (num_edges - local_start_edge)) ? local_num_edges : (num_edges - local_start_edge); + local_num_vertices = (local_num_vertices < (num_vertices - local_start_vertex)) ? local_num_vertices : (num_vertices - local_start_vertex); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + size_t vertex_count = (local_num_vertices + num_local_arrays - 1) / num_local_arrays; + size_t vertex_start = i * vertex_count; + vertex_count = (vertex_count < (local_num_vertices - vertex_start)) ? vertex_count : (local_num_vertices - vertex_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, vertex_count, vertex_tid, vertices + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + size_t edge_count = (local_num_edges + num_local_arrays - 1) / num_local_arrays; + size_t edge_start = i * edge_count; + edge_count = (edge_count < (local_num_edges - edge_start)) ? edge_count : (local_num_edges - edge_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, src + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, dst + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, weight_tid, wgt + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view[i] = cugraph_type_erased_device_array_view(vertices[i]); + src_view[i] = cugraph_type_erased_device_array_view(src[i]); + dst_view[i] = cugraph_type_erased_device_array_view(dst[i]); + wgt_view[i] = cugraph_type_erased_device_array_view(wgt[i]); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view[i], (byte_t*)(h_vertices + local_start_vertex + vertex_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view[i], (byte_t*)(h_src + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view[i], (byte_t*)(h_dst + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view[i], (byte_t*)(h_wgt + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + } + + ret_code = cugraph_graph_create_mg(handle, + &properties, + (cugraph_type_erased_device_array_view_t const* const*) vertices_view, + (cugraph_type_erased_device_array_view_t const* const*) src_view, + (cugraph_type_erased_device_array_view_t const* const*) dst_view, + (cugraph_type_erased_device_array_view_t const* const*) wgt_view, + NULL, + NULL, + FALSE, + num_local_arrays, + TRUE, + TRUE, + TRUE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + // + // Now call pagerank and check results... + // + cugraph_centrality_result_t* result = NULL; + + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + + // NOTE: Because we get back vertex ids and pageranks, we can simply compare + // the returned values with the expected results for the entire + // graph. Each GPU will have a subset of the total vertices, so + // they will do a subset of the comparisons. + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(result_vertices); + + vertex_t h_result_vertices[num_local_vertices]; + weight_t h_pageranks[num_local_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + cugraph_type_erased_device_array_view_free(wgt_view[i]); + cugraph_type_erased_device_array_view_free(dst_view[i]); + cugraph_type_erased_device_array_view_free(src_view[i]); + cugraph_type_erased_device_array_view_free(vertices_view[i]); + cugraph_type_erased_device_array_free(wgt[i]); + cugraph_type_erased_device_array_free(dst[i]); + cugraph_type_erased_device_array_free(src[i]); + cugraph_type_erased_device_array_free(vertices[i]); + } + + cugraph_error_free(ret_error); + + return test_ret_value; +} + /******************************************************************************/ int main(int argc, char** argv) @@ -131,6 +513,8 @@ int main(int argc, char** argv) int result = 0; result |= RUN_MG_TEST(test_create_mg_graph_simple, handle); + result |= RUN_MG_TEST(test_create_mg_graph_multiple_edge_lists, handle); + result |= RUN_MG_TEST(test_create_mg_graph_multiple_edge_lists_multi_edge, handle); cugraph_free_resource_handle(handle); free_mg_raft_handle(raft_handle); diff --git a/cpp/tests/c_api/mg_test_utils.cpp b/cpp/tests/c_api/mg_test_utils.cpp index 15df613ae05..6eec436e77d 100644 --- a/cpp/tests/c_api/mg_test_utils.cpp +++ b/cpp/tests/c_api/mg_test_utils.cpp @@ -158,30 +158,22 @@ extern "C" int create_mg_test_graph(const cugraph_resource_handle_t* handle, rank = cugraph_resource_handle_get_rank(handle); - if (rank == 0) { - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); + size_t original_num_edges = num_edges; - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + if (rank != 0) num_edges = 0; - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); - } else { - ret_code = cugraph_type_erased_device_array_create(handle, 0, vertex_tid, &src, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); - ret_code = cugraph_type_erased_device_array_create(handle, 0, vertex_tid, &dst, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); - ret_code = cugraph_type_erased_device_array_create(handle, 0, weight_tid, &wgt, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); - } + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); src_view = cugraph_type_erased_device_array_view(src); dst_view = cugraph_type_erased_device_array_view(dst); @@ -207,7 +199,7 @@ extern "C" int create_mg_test_graph(const cugraph_resource_handle_t* handle, NULL, NULL, store_transposed, - num_edges, + original_num_edges, // UNUSED FALSE, p_graph, ret_error); @@ -260,30 +252,22 @@ extern "C" int create_mg_test_graph_double(const cugraph_resource_handle_t* hand rank = cugraph_resource_handle_get_rank(handle); - if (rank == 0) { - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); + size_t original_num_edges = num_edges; - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + if (rank != 0) num_edges = 0; - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); - } else { - ret_code = cugraph_type_erased_device_array_create(handle, 0, vertex_tid, &src, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); - ret_code = cugraph_type_erased_device_array_create(handle, 0, vertex_tid, &dst, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); - ret_code = cugraph_type_erased_device_array_create(handle, 0, weight_tid, &wgt, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); - } + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); src_view = cugraph_type_erased_device_array_view(src); dst_view = cugraph_type_erased_device_array_view(dst); @@ -309,7 +293,7 @@ extern "C" int create_mg_test_graph_double(const cugraph_resource_handle_t* hand NULL, NULL, store_transposed, - num_edges, + original_num_edges, // UNUSED FALSE, p_graph, ret_error); @@ -357,30 +341,22 @@ extern "C" int create_mg_test_graph_with_edge_ids(const cugraph_resource_handle_ rank = cugraph_resource_handle_get_rank(handle); - if (rank == 0) { - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); + size_t original_num_edges = num_edges; - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + if (rank != 0) num_edges = 0; - ret_code = - cugraph_type_erased_device_array_create(handle, num_edges, edge_tid, &idx, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "idx create failed."); - } else { - ret_code = cugraph_type_erased_device_array_create(handle, 0, vertex_tid, &src, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); - ret_code = cugraph_type_erased_device_array_create(handle, 0, vertex_tid, &dst, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); - ret_code = cugraph_type_erased_device_array_create(handle, 0, edge_tid, &idx, ret_error); - TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); - } + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, edge_tid, &idx, ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "idx create failed."); src_view = cugraph_type_erased_device_array_view(src); dst_view = cugraph_type_erased_device_array_view(dst); @@ -406,7 +382,7 @@ extern "C" int create_mg_test_graph_with_edge_ids(const cugraph_resource_handle_ idx_view, NULL, store_transposed, - num_edges, + original_num_edges, // UNUSED FALSE, p_graph, ret_error); @@ -464,7 +440,7 @@ extern "C" int create_mg_test_graph_with_properties(const cugraph_resource_handl size_t original_num_edges = num_edges; - if (rank == 0) num_edges = 0; + if (rank != 0) num_edges = 0; ret_code = cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); @@ -529,7 +505,7 @@ extern "C" int create_mg_test_graph_with_properties(const cugraph_resource_handl idx_view, type_view, store_transposed, - original_num_edges, + original_num_edges, // UNUSED FALSE, p_graph, ret_error); @@ -593,7 +569,7 @@ int create_mg_test_graph_new(const cugraph_resource_handle_t* handle, size_t original_num_edges = num_edges; if (rank != 0) num_edges = 0; - + ret_code = cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); @@ -658,7 +634,7 @@ int create_mg_test_graph_new(const cugraph_resource_handle_t* handle, edge_id_view, edge_type_view, store_transposed, - renumber, + original_num_edges, // UNUSED FALSE, graph, ret_error); diff --git a/cpp/tests/centrality/eigenvector_centrality_test.cpp b/cpp/tests/centrality/eigenvector_centrality_test.cpp index 7cafcfbde85..6c3bd510abd 100644 --- a/cpp/tests/centrality/eigenvector_centrality_test.cpp +++ b/cpp/tests/centrality/eigenvector_centrality_test.cpp @@ -60,7 +60,6 @@ void eigenvector_centrality_reference(vertex_t const* src, size_t iter{0}; while (true) { std::copy(tmp_centralities.begin(), tmp_centralities.end(), old_centralities.begin()); - std::fill(tmp_centralities.begin(), tmp_centralities.end(), double{0}); for (size_t e = 0; e < num_edges; ++e) { auto w = weights ? (*weights)[e] : weight_t{1.0}; diff --git a/cpp/tests/community/louvain_test.cpp b/cpp/tests/community/louvain_test.cpp index 1e1fb6d4c33..284dcc94b8c 100644 --- a/cpp/tests/community/louvain_test.cpp +++ b/cpp/tests/community/louvain_test.cpp @@ -317,72 +317,6 @@ TEST(louvain_legacy, success) } } -TEST(louvain_legacy_renumbered, success) -{ - raft::handle_t handle; - - auto stream = handle.get_stream(); - - std::vector off_h = {0, 16, 25, 30, 34, 38, 42, 44, 46, 48, 50, 52, - 54, 56, 73, 85, 95, 101, 107, 112, 117, 121, 125, 129, - 132, 135, 138, 141, 144, 147, 149, 151, 153, 155, 156}; - std::vector ind_h = { - 1, 3, 7, 11, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 30, 33, 0, 5, 11, 15, 16, 19, 21, - 25, 30, 4, 13, 14, 22, 27, 0, 9, 20, 24, 2, 13, 15, 26, 1, 13, 14, 18, 13, 15, 0, 16, - 13, 14, 3, 20, 13, 14, 0, 1, 13, 22, 2, 4, 5, 6, 8, 10, 12, 14, 17, 18, 19, 22, 25, - 28, 29, 31, 32, 2, 5, 8, 10, 13, 15, 17, 18, 22, 29, 31, 32, 0, 1, 4, 6, 14, 16, 18, - 19, 21, 28, 0, 1, 7, 15, 19, 21, 0, 13, 14, 26, 27, 28, 0, 5, 13, 14, 15, 0, 1, 13, - 16, 16, 0, 3, 9, 23, 0, 1, 15, 16, 2, 12, 13, 14, 0, 20, 24, 0, 3, 23, 0, 1, 13, - 4, 17, 27, 2, 17, 26, 13, 15, 17, 13, 14, 0, 1, 13, 14, 13, 14, 0}; - - std::vector w_h = { - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; - - int num_verts = off_h.size() - 1; - int num_edges = ind_h.size(); - - rmm::device_uvector offsets_v(num_verts + 1, stream); - rmm::device_uvector indices_v(num_edges, stream); - rmm::device_uvector weights_v(num_edges, stream); - rmm::device_uvector result_v(num_verts, stream); - - raft::update_device(offsets_v.data(), off_h.data(), off_h.size(), stream); - raft::update_device(indices_v.data(), ind_h.data(), ind_h.size(), stream); - raft::update_device(weights_v.data(), w_h.data(), w_h.size(), stream); - - cugraph::legacy::GraphCSRView G( - offsets_v.data(), indices_v.data(), weights_v.data(), num_verts, num_edges); - - float modularity{0.0}; - size_t num_level = 40; - - // "FIXME": remove this check once we drop support for Pascal - // - // Calling louvain on Pascal will throw an exception, we'll check that - // this is the behavior while we still support Pascal (device_prop.major < 7) - // - if (handle.get_device_properties().major < 7) { - EXPECT_THROW(cugraph::louvain(handle, G, result_v.data()), cugraph::logic_error); - } else { - std::tie(num_level, modularity) = cugraph::louvain(handle, G, result_v.data()); - - auto cluster_id = cugraph::test::to_host(handle, result_v); - - int min = *min_element(cluster_id.begin(), cluster_id.end()); - - ASSERT_GE(min, 0); - ASSERT_FLOAT_EQ(modularity, 0.41880345); - } -} - using Tests_Louvain_File = Tests_Louvain; using Tests_Louvain_File32 = Tests_Louvain; using Tests_Louvain_File64 = Tests_Louvain; @@ -390,11 +324,15 @@ using Tests_Louvain_Rmat = Tests_Louvain; using Tests_Louvain_Rmat32 = Tests_Louvain; using Tests_Louvain_Rmat64 = Tests_Louvain; +#if 0 +// FIXME: Reenable legacy tests once threshold parameter is exposed +// by louvain legacy API. TEST_P(Tests_Louvain_File, CheckInt32Int32FloatFloatLegacy) { run_legacy_test( override_File_Usecase_with_cmd_line_arguments(GetParam())); } +#endif TEST_P(Tests_Louvain_File, CheckInt32Int32FloatFloat) { @@ -458,11 +396,12 @@ TEST_P(Tests_Louvain_Rmat64, CheckInt64Int64FloatFloat) INSTANTIATE_TEST_SUITE_P( simple_test, Tests_Louvain_File, - ::testing::Combine( - ::testing::Values(Louvain_Usecase{std::nullopt, std::nullopt, std::nullopt, true, 3, 0.408695}, - Louvain_Usecase{20, double{1e-4}, std::nullopt, true, 3, 0.408695}, - Louvain_Usecase{100, double{1e-4}, double{0.8}, true, 3, 0.48336622}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + ::testing::Combine(::testing::Values( + Louvain_Usecase{ + std::nullopt, std::nullopt, std::nullopt, true, 3, 0.39907956}, + Louvain_Usecase{20, double{1e-3}, std::nullopt, true, 3, 0.39907956}, + Louvain_Usecase{100, double{1e-3}, double{0.8}, true, 3, 0.47547662}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); INSTANTIATE_TEST_SUITE_P( file_benchmark_test, /* note that the test filename can be overridden in benchmarking (with diff --git a/cpp/tests/community/mg_egonet_test.cu b/cpp/tests/community/mg_egonet_test.cu index 42a2bba1181..6660eac3cad 100644 --- a/cpp/tests/community/mg_egonet_test.cu +++ b/cpp/tests/community/mg_egonet_test.cu @@ -215,6 +215,7 @@ class Tests_MGEgonet graph_ids_v.end(), size_t{0}, d_mg_edgelist_offsets.size() - 1, + true, handle_->get_stream()); auto [d_reference_src, d_reference_dst, d_reference_wgt, d_reference_offsets] = diff --git a/cpp/tests/link_analysis/hits_test.cpp b/cpp/tests/link_analysis/hits_test.cpp index 44fa619b503..d0e77769034 100644 --- a/cpp/tests/link_analysis/hits_test.cpp +++ b/cpp/tests/link_analysis/hits_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -176,7 +176,7 @@ class Tests_Hits : public ::testing::TestWithParam d_hubs(graph_view.local_vertex_partition_range_size(), handle.get_stream()); diff --git a/cpp/tests/mtmg/multi_node_threaded_test.cu b/cpp/tests/mtmg/multi_node_threaded_test.cu new file mode 100644 index 00000000000..e5a7de07781 --- /dev/null +++ b/cpp/tests/mtmg/multi_node_threaded_test.cu @@ -0,0 +1,516 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include +#include +#include + +#include +#include + +struct Multithreaded_Usecase { + bool test_weighted{false}; + bool check_correctness{true}; +}; + +// Global variable defining resource manager +static cugraph::mtmg::resource_manager_t g_resource_manager{}; +static int g_node_rank{-1}; +static int g_num_nodes{-1}; + +template +class Tests_Multithreaded + : public ::testing::TestWithParam> { + public: + Tests_Multithreaded() {} + + virtual void SetUp() {} + virtual void TearDown() {} + + std::vector get_gpu_list() + { + int num_gpus_per_node{1}; + RAFT_CUDA_TRY(cudaGetDeviceCount(&num_gpus_per_node)); + + std::vector gpu_list(num_gpus_per_node); + std::iota(gpu_list.begin(), gpu_list.end(), 0); + + return gpu_list; + } + + template + void run_current_test( + std::tuple const& param, + std::vector gpu_list) + { + using edge_type_t = int32_t; + + constexpr bool renumber = true; + constexpr bool do_expensive_check = false; + + auto [multithreaded_usecase, input_usecase] = param; + + raft::handle_t handle{}; + + result_t constexpr alpha{0.85}; + result_t constexpr epsilon{1e-6}; + + size_t device_buffer_size{64 * 1024 * 1024}; + size_t thread_buffer_size{4 * 1024 * 1024}; + + int num_local_gpus = gpu_list.size(); + int num_threads = num_local_gpus * 4; + + ncclUniqueId instance_manager_id{}; + + if (g_node_rank == 0) RAFT_NCCL_TRY(ncclGetUniqueId(&instance_manager_id)); + + RAFT_MPI_TRY( + MPI_Bcast(&instance_manager_id, sizeof(instance_manager_id), MPI_CHAR, 0, MPI_COMM_WORLD)); + + auto instance_manager = g_resource_manager.create_instance_manager( + g_resource_manager.registered_ranks(), instance_manager_id); + + cugraph::mtmg::edgelist_t edgelist; + cugraph::mtmg::graph_t graph; + cugraph::mtmg::graph_view_t graph_view; + cugraph::mtmg::vertex_result_t pageranks; + std::optional> renumber_map = + std::make_optional>(); + + auto edge_weights = multithreaded_usecase.test_weighted + ? std::make_optional, + weight_t>>() + : std::nullopt; + + // + // Simulate graph creation by spawning threads to walk through the + // local COO and add edges + // + std::vector running_threads; + + // Initialize shared edgelist object, one per GPU + for (int i = 0; i < num_local_gpus; ++i) { + running_threads.emplace_back([&instance_manager, + &edgelist, + device_buffer_size, + use_weight = true, + use_edge_id = false, + use_edge_type = false]() { + auto thread_handle = instance_manager->get_handle(); + + edgelist.set(thread_handle, device_buffer_size, use_weight, use_edge_id, use_edge_type); + }); + } + + // Wait for CPU threads to complete + std::for_each(running_threads.begin(), running_threads.end(), [](auto& t) { t.join(); }); + running_threads.resize(0); + instance_manager->reset_threads(); + + // Load SG edge list + auto [d_src_v, d_dst_v, d_weights_v, d_vertices_v, is_symmetric] = + input_usecase.template construct_edgelist( + handle, multithreaded_usecase.test_weighted, false, false); + + auto h_src_v = cugraph::test::to_host(handle, d_src_v); + auto h_dst_v = cugraph::test::to_host(handle, d_dst_v); + auto h_weights_v = cugraph::test::to_host(handle, d_weights_v); + auto unique_vertices = cugraph::test::to_host(handle, d_vertices_v); + + // Load edgelist from different threads. We'll use more threads than GPUs here + for (int i = 0; i < num_threads; ++i) { + running_threads.emplace_back([&instance_manager, + thread_buffer_size, + &edgelist, + &h_src_v, + &h_dst_v, + &h_weights_v, + starting_edge_offset = g_node_rank * num_threads + i, + stride = g_num_nodes * num_threads]() { + auto thread_handle = instance_manager->get_handle(); + cugraph::mtmg::per_thread_edgelist_t + per_thread_edgelist(edgelist.get(thread_handle), thread_buffer_size); + + for (size_t j = starting_edge_offset; j < h_src_v.size(); j += stride) { + per_thread_edgelist.append( + thread_handle, + h_src_v[j], + h_dst_v[j], + h_weights_v ? std::make_optional((*h_weights_v)[j]) : std::nullopt, + std::nullopt, + std::nullopt); + } + + per_thread_edgelist.flush(thread_handle); + }); + } + + // Wait for CPU threads to complete + std::for_each(running_threads.begin(), running_threads.end(), [](auto& t) { t.join(); }); + running_threads.resize(0); + instance_manager->reset_threads(); + + for (int i = 0; i < num_local_gpus; ++i) { + running_threads.emplace_back([&instance_manager, + &graph, + &edge_weights, + &edgelist, + &renumber_map, + &pageranks, + &h_src_v, // debugging + is_symmetric = is_symmetric, + renumber, + do_expensive_check]() { + auto thread_handle = instance_manager->get_handle(); + + if (thread_handle.get_thread_rank() > 0) return; + + std::optional, + edge_t>> + edge_ids{std::nullopt}; + std::optional, + int32_t>> + edge_types{std::nullopt}; + + edgelist.finalize_buffer(thread_handle); + edgelist.consolidate_and_shuffle(thread_handle, true); + + cugraph::mtmg:: + create_graph_from_edgelist( + thread_handle, + edgelist, + cugraph::graph_properties_t{is_symmetric, true}, + renumber, + graph, + edge_weights, + edge_ids, + edge_types, + renumber_map, + do_expensive_check); + }); + } + + // Wait for CPU threads to complete + std::for_each(running_threads.begin(), running_threads.end(), [](auto& t) { t.join(); }); + running_threads.resize(0); + instance_manager->reset_threads(); + + graph_view = graph.view(); + + for (int i = 0; i < num_threads; ++i) { + running_threads.emplace_back( + [&instance_manager, &graph_view, &edge_weights, &pageranks, alpha, epsilon]() { + auto thread_handle = instance_manager->get_handle(); + + if (thread_handle.get_thread_rank() > 0) return; + + auto [local_pageranks, metadata] = + cugraph::pagerank( + thread_handle.raft_handle(), + graph_view.get(thread_handle), + edge_weights ? std::make_optional(edge_weights->get(thread_handle).view()) + : std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + alpha, + epsilon, + 500, + true); + + pageranks.set(thread_handle, std::move(local_pageranks)); + }); + } + + // Wait for CPU threads to complete + std::for_each(running_threads.begin(), running_threads.end(), [](auto& t) { t.join(); }); + running_threads.resize(0); + instance_manager->reset_threads(); + + std::vector, std::vector>> computed_pageranks_v; + std::mutex computed_pageranks_lock{}; + + auto pageranks_view = pageranks.view(); + auto renumber_map_view = renumber_map ? std::make_optional(renumber_map->view()) : std::nullopt; + + // Load computed_pageranks from different threads. + for (int i = 0; i < num_local_gpus; ++i) { + running_threads.emplace_back([&instance_manager, + &graph_view, + &renumber_map_view, + &pageranks_view, + &computed_pageranks_lock, + &computed_pageranks_v, + &h_src_v, + &h_dst_v, + &h_weights_v, + &unique_vertices, + i, + num_threads]() { + auto thread_handle = instance_manager->get_handle(); + + auto number_of_vertices = unique_vertices->size(); + + std::vector my_vertex_list; + my_vertex_list.reserve((number_of_vertices + num_threads - 1) / num_threads); + + for (size_t j = i; j < number_of_vertices; j += num_threads) { + my_vertex_list.push_back((*unique_vertices)[j]); + } + + rmm::device_uvector d_my_vertex_list(my_vertex_list.size(), + thread_handle.raft_handle().get_stream()); + raft::update_device(d_my_vertex_list.data(), + my_vertex_list.data(), + my_vertex_list.size(), + thread_handle.raft_handle().get_stream()); + + auto d_my_pageranks = pageranks_view.gather( + thread_handle, + raft::device_span{d_my_vertex_list.data(), d_my_vertex_list.size()}, + graph_view, + renumber_map_view); + + std::vector my_pageranks(d_my_pageranks.size()); + raft::update_host(my_pageranks.data(), + d_my_pageranks.data(), + d_my_pageranks.size(), + thread_handle.raft_handle().get_stream()); + + { + std::lock_guard lock(computed_pageranks_lock); + computed_pageranks_v.push_back( + std::make_tuple(std::move(my_vertex_list), std::move(my_pageranks))); + } + }); + } + + // Wait for CPU threads to complete + std::for_each(running_threads.begin(), running_threads.end(), [](auto& t) { t.join(); }); + running_threads.resize(0); + instance_manager->reset_threads(); + + if (multithreaded_usecase.check_correctness) { + // Want to compare the results in computed_pageranks_v with SG results + cugraph::graph_t sg_graph(handle); + std::optional< + cugraph::edge_property_t, weight_t>> + sg_edge_weights{std::nullopt}; + std::optional> sg_renumber_map{std::nullopt}; + + std::tie(sg_graph, sg_edge_weights, std::ignore, std::ignore, sg_renumber_map) = cugraph:: + create_graph_from_edgelist( + handle, + std::nullopt, + std::move(d_src_v), + std::move(d_dst_v), + std::move(d_weights_v), + std::nullopt, + std::nullopt, + cugraph::graph_properties_t{is_symmetric, true}, + true); + + auto [sg_pageranks, meta] = cugraph::pagerank( + handle, + sg_graph.view(), + sg_edge_weights ? std::make_optional(sg_edge_weights->view()) : std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + alpha, + epsilon); + + auto h_sg_pageranks = cugraph::test::to_host(handle, sg_pageranks); + auto h_sg_renumber_map = cugraph::test::to_host(handle, sg_renumber_map); + auto compare_functor = cugraph::test::nearly_equal{ + weight_t{1e-3}, + weight_t{(weight_t{1} / static_cast(h_sg_pageranks.size())) * weight_t{1e-3}}}; + + std::for_each(computed_pageranks_v.begin(), + computed_pageranks_v.end(), + [h_sg_pageranks, compare_functor, h_sg_renumber_map](auto t1) { + std::for_each( + thrust::make_zip_iterator(std::get<0>(t1).begin(), std::get<1>(t1).begin()), + thrust::make_zip_iterator(std::get<0>(t1).end(), std::get<1>(t1).end()), + [h_sg_pageranks, compare_functor, h_sg_renumber_map](auto t2) { + vertex_t v = thrust::get<0>(t2); + weight_t pr = thrust::get<1>(t2); + + auto pos = + std::find(h_sg_renumber_map->begin(), h_sg_renumber_map->end(), v); + auto offset = std::distance(h_sg_renumber_map->begin(), pos); + + if (pos == h_sg_renumber_map->end()) { + ASSERT_TRUE(compare_functor(pr, weight_t{0})) + << "vertex " << v << ", SG result = " << h_sg_pageranks[offset] + << ", mtmg result = " << pr << ", not in renumber map"; + } else { + ASSERT_TRUE(compare_functor(pr, h_sg_pageranks[offset])) + << "vertex " << v << ", SG result = " << h_sg_pageranks[offset] + << ", mtmg result = " << pr + << ", renumber map = " << (*h_sg_renumber_map)[offset]; + } + }); + }); + } + } +}; + +using Tests_Multithreaded_File = Tests_Multithreaded; +using Tests_Multithreaded_Rmat = Tests_Multithreaded; + +// FIXME: add tests for type combinations +TEST_P(Tests_Multithreaded_File, CheckInt32Int32FloatFloat) +{ + run_current_test( + override_File_Usecase_with_cmd_line_arguments(GetParam()), get_gpu_list()); +} + +TEST_P(Tests_Multithreaded_Rmat, CheckInt32Int32FloatFloat) +{ + run_current_test( + override_Rmat_Usecase_with_cmd_line_arguments(GetParam()), get_gpu_list()); +} + +INSTANTIATE_TEST_SUITE_P(file_test, + Tests_Multithreaded_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Multithreaded_Usecase{false, true}, + Multithreaded_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("karate.csv"), + cugraph::test::File_Usecase("dolphins.csv")))); + +INSTANTIATE_TEST_SUITE_P( + rmat_small_test, + Tests_Multithreaded_Rmat, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Multithreaded_Usecase{false, true}, Multithreaded_Usecase{true, true}), + ::testing::Values(cugraph::test::Rmat_Usecase(10, 16, 0.57, 0.19, 0.19, 0, false, false)))); + +INSTANTIATE_TEST_SUITE_P( + file_benchmark_test, /* note that the test filename can be overridden in benchmarking (with + --gtest_filter to select only the file_benchmark_test with a specific + vertex & edge type combination) by command line arguments and do not + include more than one File_Usecase that differ only in filename + (to avoid running same benchmarks more than once) */ + Tests_Multithreaded_File, + ::testing::Combine( + // disable correctness checks + ::testing::Values(Multithreaded_Usecase{false, false}, Multithreaded_Usecase{true, false}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + rmat_benchmark_test, /* note that scale & edge factor can be overridden in benchmarking (with + --gtest_filter to select only the rmat_benchmark_test with a specific + vertex & edge type combination) by command line arguments and do not + include more than one Rmat_Usecase that differ only in scale or edge + factor (to avoid running same benchmarks more than once) */ + Tests_Multithreaded_Rmat, + ::testing::Combine( + // disable correctness checks for large graphs + ::testing::Values(Multithreaded_Usecase{false, false}, Multithreaded_Usecase{true, false}), + ::testing::Values(cugraph::test::Rmat_Usecase(10, 16, 0.57, 0.19, 0.19, 0, false, false)))); + +// +// Need to customize the test configuration to support multi-node comms not using MPI +// +int main(int argc, char** argv) +{ + cugraph::test::initialize_mpi(argc, argv); + auto comm_rank = cugraph::test::query_mpi_comm_world_rank(); + auto comm_size = cugraph::test::query_mpi_comm_world_size(); + + ::testing::InitGoogleTest(&argc, argv); + auto const cmd_opts = parse_test_options(argc, argv); + auto const rmm_mode = cmd_opts["rmm_mode"].as(); + auto resource = cugraph::test::create_memory_resource(rmm_mode); + rmm::mr::set_current_device_resource(resource.get()); + cugraph::test::g_perf = cmd_opts["perf"].as(); + cugraph::test::g_rmat_scale = (cmd_opts.count("rmat_scale") > 0) + ? std::make_optional(cmd_opts["rmat_scale"].as()) + : std::nullopt; + cugraph::test::g_rmat_edge_factor = + (cmd_opts.count("rmat_edge_factor") > 0) + ? std::make_optional(cmd_opts["rmat_edge_factor"].as()) + : std::nullopt; + cugraph::test::g_test_file_name = + (cmd_opts.count("test_file_name") > 0) + ? std::make_optional(cmd_opts["test_file_name"].as()) + : std::nullopt; + + // + // Set global values for the test. Need to know the rank of this process, + // the comm size, number of GPUs per node, and the NCCL Id for rank 0. + // + int num_gpus_this_node{-1}; + std::vector num_gpus_per_node{}; + + g_node_rank = comm_rank; + g_num_nodes = comm_size; + + num_gpus_per_node.resize(comm_size); + + RAFT_CUDA_TRY(cudaGetDeviceCount(&num_gpus_this_node)); + RAFT_MPI_TRY(MPI_Allgather( + &num_gpus_this_node, 1, MPI_INT, num_gpus_per_node.data(), 1, MPI_INT, MPI_COMM_WORLD)); + + int node_rank{0}; + + for (int i = 0; i < comm_size; ++i) { + for (int j = 0; j < num_gpus_per_node[i]; ++j) { + if (i != comm_rank) + g_resource_manager.register_remote_gpu(node_rank++); + else + g_resource_manager.register_local_gpu(node_rank++, rmm::cuda_device_id{j}); + } + } + + auto result = RUN_ALL_TESTS(); + cugraph::test::finalize_mpi(); + return result; +} diff --git a/cpp/tests/mtmg/threaded_test.cu b/cpp/tests/mtmg/threaded_test.cu index c5dc2d3c7ce..bc4d8cfef6a 100644 --- a/cpp/tests/mtmg/threaded_test.cu +++ b/cpp/tests/mtmg/threaded_test.cu @@ -94,8 +94,9 @@ class Tests_Multithreaded size_t device_buffer_size{64 * 1024 * 1024}; size_t thread_buffer_size{4 * 1024 * 1024}; + const int num_threads_per_gpu{4}; int num_gpus = gpu_list.size(); - int num_threads = num_gpus * 4; + int num_threads = num_gpus * num_threads_per_gpu; cugraph::mtmg::resource_manager_t resource_manager; @@ -106,8 +107,10 @@ class Tests_Multithreaded ncclUniqueId instance_manager_id; ncclGetUniqueId(&instance_manager_id); + // Currently the only uses for multiple streams for each CPU threads + // associated with a particular GPU, which is a constant set above auto instance_manager = resource_manager.create_instance_manager( - resource_manager.registered_ranks(), instance_manager_id); + resource_manager.registered_ranks(), instance_manager_id, num_threads_per_gpu); cugraph::mtmg::edgelist_t edgelist; cugraph::mtmg::graph_t graph; @@ -172,15 +175,6 @@ class Tests_Multithreaded per_thread_edgelist(edgelist.get(thread_handle), thread_buffer_size); for (size_t j = i; j < h_src_v.size(); j += num_threads) { -#if 0 - if (h_weights_v) { - thread_edgelist.append( - thread_handle, h_src_v[j], h_dst_v[j], (*h_weights_v)[j], std::nullopt, std::nullopt); - } else { - thread_edgelist.append( - thread_handle, h_src_v[j], h_dst_v[j], std::nullopt, std::nullopt, std::nullopt); - } -#endif per_thread_edgelist.append( thread_handle, h_src_v[j], diff --git a/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_intersection.cu b/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_intersection.cu index a7cd8a989b0..a3edb1f6372 100644 --- a/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_intersection.cu +++ b/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_intersection.cu @@ -21,9 +21,13 @@ #include #include +#include +#include #include +#include #include +#include #include #include #include @@ -61,6 +65,7 @@ struct intersection_op_t { struct Prims_Usecase { size_t num_vertex_pairs{0}; + bool edge_masking{false}; bool check_correctness{true}; }; @@ -78,7 +83,7 @@ class Tests_MGPerVPairTransformDstNbrIntersection virtual void TearDown() {} // Verify the results of per_v_pair_transform_dst_nbr_intersection primitive - template + template void run_current_test(Prims_Usecase const& prims_usecase, input_usecase_t const& input_usecase) { HighResTimer hr_timer{}; @@ -109,6 +114,34 @@ class Tests_MGPerVPairTransformDstNbrIntersection auto mg_graph_view = mg_graph.view(); + std::optional> edge_mask{std::nullopt}; + if (prims_usecase.edge_masking) { + cugraph::edge_src_property_t edge_src_renumber_map( + *handle_, mg_graph_view); + cugraph::edge_dst_property_t edge_dst_renumber_map( + *handle_, mg_graph_view); + cugraph::update_edge_src_property( + *handle_, mg_graph_view, (*mg_renumber_map).begin(), edge_src_renumber_map); + cugraph::update_edge_dst_property( + *handle_, mg_graph_view, (*mg_renumber_map).begin(), edge_dst_renumber_map); + + edge_mask = cugraph::edge_property_t(*handle_, mg_graph_view); + + cugraph::transform_e( + *handle_, + mg_graph_view, + edge_src_renumber_map.view(), + edge_dst_renumber_map.view(), + cugraph::edge_dummy_property_t{}.view(), + [] __device__(auto src, auto dst, auto src_property, auto dst_property, thrust::nullopt_t) { + return ((src_property % 2 == 0) && (dst_property % 2 == 0)) + ? false + : true; // mask out the edges with even unrenumbered src & dst vertex IDs + }, + (*edge_mask).mutable_view()); + mg_graph_view.attach_edge_mask((*edge_mask).view()); + } + // 2. run MG per_v_pair_transform_dst_nbr_intersection primitive ASSERT_TRUE( @@ -224,6 +257,42 @@ class Tests_MGPerVPairTransformDstNbrIntersection if (handle_->get_comms().get_rank() == 0) { auto sg_graph_view = sg_graph.view(); + if (prims_usecase.edge_masking) { + rmm::device_uvector srcs(0, handle_->get_stream()); + rmm::device_uvector dsts(0, handle_->get_stream()); + std::tie(srcs, dsts, std::ignore, std::ignore) = + cugraph::decompress_to_edgelist( + *handle_, sg_graph_view, std::nullopt, std::nullopt, std::nullopt); + auto edge_first = thrust::make_zip_iterator(srcs.begin(), dsts.begin()); + srcs.resize(thrust::distance(edge_first, + thrust::remove_if(handle_->get_thrust_policy(), + edge_first, + edge_first + srcs.size(), + [] __device__(auto pair) { + return (thrust::get<0>(pair) % 2 == 0) && + (thrust::get<1>(pair) % 2 == 0); + })), + handle_->get_stream()); + dsts.resize(srcs.size(), handle_->get_stream()); + rmm::device_uvector vertices(sg_graph_view.number_of_vertices(), + handle_->get_stream()); + thrust::sequence( + handle_->get_thrust_policy(), vertices.begin(), vertices.end(), vertex_t{0}); + std::tie(sg_graph, std::ignore, std::ignore, std::ignore, std::ignore) = cugraph:: + create_graph_from_edgelist( + *handle_, + std::move(vertices), + std::move(srcs), + std::move(dsts), + std::nullopt, + std::nullopt, + std::nullopt, + cugraph::graph_properties_t{sg_graph_view.is_symmetric(), + sg_graph_view.is_multigraph()}, + false); + sg_graph_view = sg_graph.view(); + } + auto sg_result_buffer = cugraph::allocate_dataframe_buffer>( cugraph::size_dataframe_buffer(mg_aggregate_vertex_pair_buffer), handle_->get_stream()); auto sg_out_degrees = sg_graph_view.compute_out_degrees(*handle_); @@ -262,47 +331,16 @@ using Tests_MGPerVPairTransformDstNbrIntersection_File = using Tests_MGPerVPairTransformDstNbrIntersection_Rmat = Tests_MGPerVPairTransformDstNbrIntersection; -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_File, CheckInt32Int32FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>(std::get<0>(param), - std::get<1>(param)); -} - -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int32FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>( - std::get<0>(param), - cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); -} - -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int64FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>( - std::get<0>(param), - cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); -} - -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt64Int64FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>( - std::get<0>(param), - cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); -} - TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_File, CheckInt32Int32Float) { auto param = GetParam(); - run_current_test(std::get<0>(param), std::get<1>(param)); + run_current_test(std::get<0>(param), std::get<1>(param)); } TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int32Float) { auto param = GetParam(); - run_current_test( + run_current_test( std::get<0>(param), cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } @@ -310,7 +348,7 @@ TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int32Float) TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int64Float) { auto param = GetParam(); - run_current_test( + run_current_test( std::get<0>(param), cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } @@ -318,7 +356,7 @@ TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int64Float) TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt64Int64Float) { auto param = GetParam(); - run_current_test( + run_current_test( std::get<0>(param), cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } @@ -327,15 +365,18 @@ INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGPerVPairTransformDstNbrIntersection_File, ::testing::Combine( - ::testing::Values(Prims_Usecase{size_t{1024}, true}), + ::testing::Values(Prims_Usecase{size_t{1024}, false, true}, + Prims_Usecase{size_t{1024}, true, true}), ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), cugraph::test::File_Usecase("test/datasets/netscience.mtx")))); -INSTANTIATE_TEST_SUITE_P(rmat_small_test, - Tests_MGPerVPairTransformDstNbrIntersection_Rmat, - ::testing::Combine(::testing::Values(Prims_Usecase{size_t{1024}, true}), - ::testing::Values(cugraph::test::Rmat_Usecase( - 10, 16, 0.57, 0.19, 0.19, 0, false, false)))); +INSTANTIATE_TEST_SUITE_P( + rmat_small_test, + Tests_MGPerVPairTransformDstNbrIntersection_Rmat, + ::testing::Combine( + ::testing::Values(Prims_Usecase{size_t{1024}, false, true}, + Prims_Usecase{size_t{1024}, true, true}), + ::testing::Values(cugraph::test::Rmat_Usecase(10, 16, 0.57, 0.19, 0.19, 0, false, false)))); INSTANTIATE_TEST_SUITE_P( rmat_benchmark_test, /* note that scale & edge factor can be overridden in benchmarking (with @@ -345,7 +386,8 @@ INSTANTIATE_TEST_SUITE_P( factor (to avoid running same benchmarks more than once) */ Tests_MGPerVPairTransformDstNbrIntersection_Rmat, ::testing::Combine( - ::testing::Values(Prims_Usecase{size_t{1024 * 1024}, false}), + ::testing::Values(Prims_Usecase{size_t{1024 * 1024}, false, false}, + Prims_Usecase{size_t{1024 * 1024}, true, false}), ::testing::Values(cugraph::test::Rmat_Usecase(20, 32, 0.57, 0.19, 0.19, 0, false, false)))); CUGRAPH_MG_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_weighted_intersection.cu b/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_weighted_intersection.cu index 3b6a6b9c4c5..4d05b0c9e65 100644 --- a/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_weighted_intersection.cu +++ b/cpp/tests/prims/mg_per_v_pair_transform_dst_nbr_weighted_intersection.cu @@ -50,14 +50,14 @@ template struct intersection_op_t { - __device__ thrust::tuple operator()( + __device__ thrust::tuple operator()( vertex_t a, vertex_t b, weight_t weight_a /* weighted out degree */, weight_t weight_b /* weighted out degree */, raft::device_span intersection, - raft::device_span intersected_properties_a, - raft::device_span intersected_properties_b) const + raft::device_span intersected_property_values_a, + raft::device_span intersected_property_values_b) const { weight_t min_weight_a_intersect_b = weight_t{0}; weight_t max_weight_a_intersect_b = weight_t{0}; @@ -65,10 +65,12 @@ struct intersection_op_t { weight_t sum_of_intersected_b = weight_t{0}; for (size_t k = 0; k < intersection.size(); k++) { - min_weight_a_intersect_b += min(intersected_properties_a[k], intersected_properties_b[k]); - max_weight_a_intersect_b += max(intersected_properties_a[k], intersected_properties_b[k]); - sum_of_intersected_a += intersected_properties_a[k]; - sum_of_intersected_b += intersected_properties_b[k]; + min_weight_a_intersect_b += + min(intersected_property_values_a[k], intersected_property_values_b[k]); + max_weight_a_intersect_b += + max(intersected_property_values_a[k], intersected_property_values_b[k]); + sum_of_intersected_a += intersected_property_values_a[k]; + sum_of_intersected_b += intersected_property_values_b[k]; } weight_t sum_of_uniq_a = weight_a - sum_of_intersected_a; @@ -99,7 +101,7 @@ class Tests_MGPerVPairTransformDstNbrIntersection virtual void TearDown() {} // Verify the results of per_v_pair_transform_dst_nbr_intersection primitive - template + template void run_current_test(Prims_Usecase const& prims_usecase, input_usecase_t const& input_usecase) { HighResTimer hr_timer{}; @@ -189,7 +191,7 @@ class Tests_MGPerVPairTransformDstNbrIntersection auto mg_result_buffer = cugraph::allocate_dataframe_buffer>( cugraph::size_dataframe_buffer(mg_vertex_pair_buffer), handle_->get_stream()); - auto mg_out_degrees = mg_graph_view.compute_out_degrees(*handle_); + auto mg_out_weight_sums = compute_out_weight_sums(*handle_, mg_graph_view, mg_edge_weight_view); if (cugraph::test::g_perf) { RAFT_CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement @@ -197,9 +199,6 @@ class Tests_MGPerVPairTransformDstNbrIntersection hr_timer.start("MG per_v_pair_transform_dst_nbr_intersection"); } - rmm::device_uvector mg_out_weight_sums = - compute_out_weight_sums(*handle_, mg_graph_view, mg_edge_weight_view); - cugraph::per_v_pair_transform_dst_nbr_intersection( *handle_, mg_graph_view, @@ -272,7 +271,6 @@ class Tests_MGPerVPairTransformDstNbrIntersection if (handle_->get_comms().get_rank() == 0) { auto sg_graph_view = sg_graph.view(); - auto sg_result_buffer = cugraph::allocate_dataframe_buffer>( cugraph::size_dataframe_buffer(mg_aggregate_vertex_pair_buffer), handle_->get_stream()); @@ -290,11 +288,21 @@ class Tests_MGPerVPairTransformDstNbrIntersection */), sg_out_weight_sums.begin(), intersection_op_t{}, cugraph::get_dataframe_buffer_begin(sg_result_buffer)); + auto threshold_ratio = weight_t{1e-4}; + auto threshold_magnitude = std::numeric_limits::min(); + auto nearly_equal = [threshold_ratio, threshold_magnitude] __device__(auto lhs, auto rhs) { + return (fabs(thrust::get<0>(lhs) - thrust::get<0>(rhs)) < + max(max(thrust::get<0>(lhs), thrust::get<0>(rhs)) * threshold_ratio, + threshold_magnitude)) && + (fabs(thrust::get<1>(lhs) - thrust::get<1>(rhs)) < + max(max(thrust::get<1>(lhs), thrust::get<1>(rhs)) * threshold_ratio, + threshold_magnitude)); + }; bool valid = thrust::equal(handle_->get_thrust_policy(), cugraph::get_dataframe_buffer_begin(mg_aggregate_result_buffer), cugraph::get_dataframe_buffer_end(mg_aggregate_result_buffer), - cugraph::get_dataframe_buffer_begin(sg_result_buffer)); - + cugraph::get_dataframe_buffer_begin(sg_result_buffer), + nearly_equal); ASSERT_TRUE(valid); } } @@ -313,47 +321,16 @@ using Tests_MGPerVPairTransformDstNbrIntersection_File = using Tests_MGPerVPairTransformDstNbrIntersection_Rmat = Tests_MGPerVPairTransformDstNbrIntersection; -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_File, CheckInt32Int32FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>(std::get<0>(param), - std::get<1>(param)); -} - -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int32FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>( - std::get<0>(param), - cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); -} - -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int64FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>( - std::get<0>(param), - cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); -} - -TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt64Int64FloatTupleIntFloat) -{ - auto param = GetParam(); - run_current_test>( - std::get<0>(param), - cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); -} - TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_File, CheckInt32Int32Float) { auto param = GetParam(); - run_current_test(std::get<0>(param), std::get<1>(param)); + run_current_test(std::get<0>(param), std::get<1>(param)); } TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int32Float) { auto param = GetParam(); - run_current_test( + run_current_test( std::get<0>(param), cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } @@ -361,7 +338,7 @@ TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int32Float) TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int64Float) { auto param = GetParam(); - run_current_test( + run_current_test( std::get<0>(param), cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } @@ -369,7 +346,7 @@ TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt32Int64Float) TEST_P(Tests_MGPerVPairTransformDstNbrIntersection_Rmat, CheckInt64Int64Float) { auto param = GetParam(); - run_current_test( + run_current_test( std::get<0>(param), cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } diff --git a/cpp/tests/prims/mg_transform_e.cu b/cpp/tests/prims/mg_transform_e.cu index 127eddd43c7..24deaad810a 100644 --- a/cpp/tests/prims/mg_transform_e.cu +++ b/cpp/tests/prims/mg_transform_e.cu @@ -51,6 +51,7 @@ #include struct Prims_Usecase { + bool use_edgelist{false}; bool check_correctness{true}; }; @@ -113,8 +114,9 @@ class Tests_MGTransformE auto mg_dst_prop = cugraph::test::generate::dst_property( *handle_, mg_graph_view, mg_vertex_prop); - cugraph::edge_bucket_t edge_list(*handle_); - { + cugraph::edge_bucket_t edge_list( + *handle_); + if (prims_usecase.use_edgelist) { rmm::device_uvector srcs(0, handle_->get_stream()); rmm::device_uvector dsts(0, handle_->get_stream()); std::tie(srcs, dsts, std::ignore, std::ignore) = cugraph::decompress_to_edgelist( @@ -154,24 +156,41 @@ class Tests_MGTransformE if (cugraph::test::g_perf) { RAFT_CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement handle_->get_comms().barrier(); - hr_timer.start("MG transform_reduce_e"); + hr_timer.start("MG transform_e"); } - cugraph::transform_e( - *handle_, - mg_graph_view, - edge_list, - mg_src_prop.view(), - mg_dst_prop.view(), - cugraph::edge_dummy_property_t{}.view(), - [] __device__(auto src, auto dst, auto src_property, auto dst_property, thrust::nullopt_t) { - if (src_property < dst_property) { - return src_property; - } else { - return dst_property; - } - }, - edge_value_output.mutable_view()); + if (prims_usecase.use_edgelist) { + cugraph::transform_e( + *handle_, + mg_graph_view, + edge_list, + mg_src_prop.view(), + mg_dst_prop.view(), + cugraph::edge_dummy_property_t{}.view(), + [] __device__(auto src, auto dst, auto src_property, auto dst_property, thrust::nullopt_t) { + if (src_property < dst_property) { + return src_property; + } else { + return dst_property; + } + }, + edge_value_output.mutable_view()); + } else { + cugraph::transform_e( + *handle_, + mg_graph_view, + mg_src_prop.view(), + mg_dst_prop.view(), + cugraph::edge_dummy_property_t{}.view(), + [] __device__(auto src, auto dst, auto src_property, auto dst_property, thrust::nullopt_t) { + if (src_property < dst_property) { + return src_property; + } else { + return dst_property; + } + }, + edge_value_output.mutable_view()); + } if (cugraph::test::g_perf) { RAFT_CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement @@ -183,24 +202,42 @@ class Tests_MGTransformE // 3. validate MG results if (prims_usecase.check_correctness) { - auto num_invalids = cugraph::count_if_e( - *handle_, - mg_graph_view, - mg_src_prop.view(), - mg_dst_prop.view(), - edge_value_output.view(), - [property_initial_value] __device__( - auto src, auto dst, auto src_property, auto dst_property, auto edge_property) { - if (((src + dst) % 2) == 0) { + size_t num_invalids{}; + if (prims_usecase.use_edgelist) { + num_invalids = cugraph::count_if_e( + *handle_, + mg_graph_view, + mg_src_prop.view(), + mg_dst_prop.view(), + edge_value_output.view(), + [property_initial_value] __device__( + auto src, auto dst, auto src_property, auto dst_property, auto edge_property) { + if (((src + dst) % 2) == 0) { + if (src_property < dst_property) { + return edge_property != src_property; + } else { + return edge_property != dst_property; + } + } else { + return edge_property != property_initial_value; + } + }); + } else { + num_invalids = cugraph::count_if_e( + *handle_, + mg_graph_view, + mg_src_prop.view(), + mg_dst_prop.view(), + edge_value_output.view(), + [property_initial_value] __device__( + auto src, auto dst, auto src_property, auto dst_property, auto edge_property) { if (src_property < dst_property) { return edge_property != src_property; } else { return edge_property != dst_property; } - } else { - return edge_property != property_initial_value; - } - }); + }); + } ASSERT_TRUE(num_invalids == 0); } @@ -278,13 +315,13 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatTupleIntFloatTransposeTrue) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } -TEST_P(Tests_MGTransformE_File, CheckInt32Int32FloatTransposeFalse) +TEST_P(Tests_MGTransformE_File, CheckInt32Int32FloatIntTransposeFalse) { auto param = GetParam(); run_current_test(std::get<0>(param), std::get<1>(param)); } -TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatTransposeFalse) +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatIntTransposeFalse) { auto param = GetParam(); run_current_test( @@ -292,7 +329,7 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatTransposeFalse) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } -TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatTransposeFalse) +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatIntTransposeFalse) { auto param = GetParam(); run_current_test( @@ -300,7 +337,7 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatTransposeFalse) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } -TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatTransposeFalse) +TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatIntTransposeFalse) { auto param = GetParam(); run_current_test( @@ -308,13 +345,13 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatTransposeFalse) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } -TEST_P(Tests_MGTransformE_File, CheckInt32Int32FloatTransposeTrue) +TEST_P(Tests_MGTransformE_File, CheckInt32Int32FloatIntTransposeTrue) { auto param = GetParam(); run_current_test(std::get<0>(param), std::get<1>(param)); } -TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatTransposeTrue) +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatIntTransposeTrue) { auto param = GetParam(); run_current_test( @@ -322,7 +359,7 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatTransposeTrue) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } -TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatTransposeTrue) +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatIntTransposeTrue) { auto param = GetParam(); run_current_test( @@ -330,7 +367,7 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatTransposeTrue) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } -TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatTransposeTrue) +TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatIntTransposeTrue) { auto param = GetParam(); run_current_test( @@ -338,11 +375,71 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatTransposeTrue) cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } +TEST_P(Tests_MGTransformE_File, CheckInt32Int32FloatBoolTransposeFalse) +{ + auto param = GetParam(); + run_current_test(std::get<0>(param), std::get<1>(param)); +} + +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatBoolTransposeFalse) +{ + auto param = GetParam(); + run_current_test( + std::get<0>(param), + cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); +} + +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatBoolTransposeFalse) +{ + auto param = GetParam(); + run_current_test( + std::get<0>(param), + cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); +} + +TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatBoolTransposeFalse) +{ + auto param = GetParam(); + run_current_test( + std::get<0>(param), + cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); +} + +TEST_P(Tests_MGTransformE_File, CheckInt32Int32FloatBoolTransposeTrue) +{ + auto param = GetParam(); + run_current_test(std::get<0>(param), std::get<1>(param)); +} + +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int32FloatBoolTransposeTrue) +{ + auto param = GetParam(); + run_current_test( + std::get<0>(param), + cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); +} + +TEST_P(Tests_MGTransformE_Rmat, CheckInt32Int64FloatBoolTransposeTrue) +{ + auto param = GetParam(); + run_current_test( + std::get<0>(param), + cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); +} + +TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatBoolTransposeTrue) +{ + auto param = GetParam(); + run_current_test( + std::get<0>(param), + cugraph::test::override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); +} + INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGTransformE_File, ::testing::Combine( - ::testing::Values(Prims_Usecase{true}), + ::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), @@ -350,7 +447,8 @@ INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(rmat_small_test, Tests_MGTransformE_Rmat, - ::testing::Combine(::testing::Values(Prims_Usecase{true}), + ::testing::Combine(::testing::Values(Prims_Usecase{false, true}, + Prims_Usecase{true, true}), ::testing::Values(cugraph::test::Rmat_Usecase( 10, 16, 0.57, 0.19, 0.19, 0, false, false)))); @@ -362,7 +460,7 @@ INSTANTIATE_TEST_SUITE_P( factor (to avoid running same benchmarks more than once) */ Tests_MGTransformE_Rmat, ::testing::Combine( - ::testing::Values(Prims_Usecase{false}), + ::testing::Values(Prims_Usecase{false, false}, Prims_Usecase{true, false}), ::testing::Values(cugraph::test::Rmat_Usecase(20, 32, 0.57, 0.19, 0.19, 0, false, false)))); CUGRAPH_MG_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/structure/graph_mask_test.cpp b/cpp/tests/structure/graph_mask_test.cpp deleted file mode 100644 index fc704d683a7..00000000000 --- a/cpp/tests/structure/graph_mask_test.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2022, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governin_from_mtxg permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -TEST(Test_GraphMask, BasicGraphMaskTestInt64) -{ - raft::handle_t handle; - - int number_of_vertices = 500; - int number_of_edges = 1000; - - cugraph::graph_mask_t mask( - handle, number_of_vertices, number_of_edges); - - auto mask_view = mask.view(); - - ASSERT_EQ(false, mask.has_vertex_mask()); - ASSERT_EQ(false, mask.has_edge_mask()); - ASSERT_EQ(false, mask_view.has_vertex_mask()); - ASSERT_EQ(false, mask_view.has_edge_mask()); - - mask.initialize_vertex_mask(); - mask.initialize_edge_mask(); - - auto mask_view2 = mask.view(); - - ASSERT_EQ(true, mask.has_vertex_mask()); - ASSERT_EQ(true, mask.has_edge_mask()); - ASSERT_EQ(true, mask_view2.has_vertex_mask()); - ASSERT_EQ(true, mask_view2.has_edge_mask()); -} \ No newline at end of file diff --git a/cpp/tests/structure/mg_induced_subgraph_test.cu b/cpp/tests/structure/mg_induced_subgraph_test.cu index 3f3db7c5278..b7bd22dfa63 100644 --- a/cpp/tests/structure/mg_induced_subgraph_test.cu +++ b/cpp/tests/structure/mg_induced_subgraph_test.cu @@ -210,6 +210,7 @@ class Tests_MGInducedSubgraph graph_ids_v.end(), size_t{0}, size_t{d_subgraph_offsets.size() - 1}, + true, handle_->get_stream()); auto [sg_graph, sg_edge_weights, sg_number_map] = cugraph::test::mg_graph_to_sg_graph( diff --git a/cpp/tests/utilities/test_utilities_impl.cuh b/cpp/tests/utilities/test_utilities_impl.cuh index 3025ca7908b..856c50ad35f 100644 --- a/cpp/tests/utilities/test_utilities_impl.cuh +++ b/cpp/tests/utilities/test_utilities_impl.cuh @@ -183,43 +183,42 @@ graph_to_host_csr( } } + auto total_global_mem = handle.get_device_properties().totalGlobalMem; + size_t element_size = sizeof(vertex_t) * 2; + if (d_wgt) { element_size += sizeof(weight_t); } + auto constexpr mem_frugal_ratio = + 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the + // total_global_mem, switch to the memory frugal approach + auto mem_frugal_threshold = + static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); + rmm::device_uvector d_offsets(0, handle.get_stream()); if (d_wgt) { std::tie(d_offsets, d_dst, *d_wgt, std::ignore) = - detail::compress_edgelist(d_src.begin(), - d_src.end(), - d_dst.begin(), - d_wgt->begin(), - vertex_t{0}, - std::optional{std::nullopt}, - graph_view.number_of_vertices(), - vertex_t{0}, - graph_view.number_of_vertices(), - handle.get_stream()); - - // segmented sort neighbors - detail::sort_adjacency_list(handle, - raft::device_span(d_offsets.data(), d_offsets.size()), - d_dst.begin(), - d_dst.end(), - d_wgt->begin()); + detail::sort_and_compress_edgelist( + std::move(d_src), + std::move(d_dst), + std::move(*d_wgt), + vertex_t{0}, + std::optional{std::nullopt}, + graph_view.number_of_vertices(), + vertex_t{0}, + graph_view.number_of_vertices(), + mem_frugal_threshold, + handle.get_stream()); } else { std::tie(d_offsets, d_dst, std::ignore) = - detail::compress_edgelist(d_src.begin(), - d_src.end(), - d_dst.begin(), - vertex_t{0}, - std::optional{std::nullopt}, - graph_view.number_of_vertices(), - vertex_t{0}, - graph_view.number_of_vertices(), - handle.get_stream()); - // segmented sort neighbors - detail::sort_adjacency_list(handle, - raft::device_span(d_offsets.data(), d_offsets.size()), - d_dst.begin(), - d_dst.end()); + detail::sort_and_compress_edgelist( + std::move(d_src), + std::move(d_dst), + vertex_t{0}, + std::optional{std::nullopt}, + graph_view.number_of_vertices(), + vertex_t{0}, + graph_view.number_of_vertices(), + mem_frugal_threshold, + handle.get_stream()); } return std::make_tuple( diff --git a/datasets/README.md b/datasets/README.md index e42413fc996..a23dc644081 100644 --- a/datasets/README.md +++ b/datasets/README.md @@ -120,9 +120,13 @@ The benchmark datasets are described below: | soc-twitter-2010 | 21,297,772 | 265,025,809 | No | No | **cit-Patents** : A citation graph that includes all citations made by patents granted between 1975 and 1999, totaling 16,522,438 citations. + **soc-LiveJournal** : A graph of the LiveJournal social network. + **europe_osm** : A graph of OpenStreetMap data for Europe. + **hollywood** : A graph of movie actors where vertices are actors, and two actors are joined by an edge whenever they appeared in a movie together. + **soc-twitter-2010** : A network of follower relationships from a snapshot of Twitter in 2010, where an edge from i to j indicates that j is a follower of i. _NOTE: the benchmark datasets were converted to a CSV format from their original format described in the reference URL below, and in doing so had edge weights and isolated vertices discarded._ diff --git a/dependencies.yaml b/dependencies.yaml index f330361ba88..2c0918ad117 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -18,6 +18,7 @@ files: - depends_on_dask_cudf - depends_on_pylibraft - depends_on_raft_dask + - depends_on_pylibcugraphops - depends_on_cupy - python_run_cugraph - python_run_nx_cugraph @@ -39,6 +40,7 @@ files: - cudatoolkit - docs - py_version + - depends_on_pylibcugraphops test_cpp: output: none includes: @@ -225,6 +227,7 @@ files: conda_dir: python/cugraph-dgl/conda includes: - checks + - depends_on_pylibcugraphops - cugraph_dgl_dev - test_python_common cugraph_pyg_dev: @@ -234,6 +237,7 @@ files: conda_dir: python/cugraph-pyg/conda includes: - checks + - depends_on_pylibcugraphops - cugraph_pyg_dev - test_python_common channels: @@ -293,10 +297,10 @@ dependencies: - cxx-compiler - gmock>=1.13.0 - gtest>=1.13.0 - - libcugraphops==23.10.* - - libraft-headers==23.10.* - - libraft==23.10.* - - librmm==23.10.* + - libcugraphops==23.12.* + - libraft-headers==23.12.* + - libraft==23.12.* + - librmm==23.12.* - openmpi # Required for building cpp-mgtests (multi-GPU tests) specific: - output_types: [conda] @@ -330,6 +334,7 @@ dependencies: common: - output_types: [conda] packages: + - breathe - doxygen - graphviz - ipython @@ -341,7 +346,6 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport - - pylibcugraphops==23.10.* py_version: specific: - output_types: [conda] @@ -373,17 +377,16 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - &dask dask==2023.9.2 - - &distributed distributed==2023.9.2 - - &dask_cuda dask-cuda==23.10.* + - &dask rapids-dask-dependency==23.12.* + - &dask_cuda dask-cuda==23.12.* - &numba numba>=0.57 - - &ucx_py ucx-py==0.34.* + - &numpy numpy>=1.21 + - &ucx_py ucx-py==0.35.* - output_types: conda packages: - aiohttp - - &dask-core_conda dask-core==2023.9.2 - fsspec>=0.6.0 - - libcudf==23.10.* + - libcudf==23.12.* - requests - nccl>=2.9.9 - ucx-proc=*=gpu @@ -397,15 +400,16 @@ dependencies: - output_types: [conda, pyproject] packages: - networkx>=3.0 + - *numpy python_run_cugraph_dgl: common: - output_types: [conda, pyproject] packages: - *numba - - &numpy numpy>=1.21 + - *numpy - output_types: [pyproject] packages: - - &cugraph cugraph==23.10.* + - &cugraph cugraph==23.12.* python_run_cugraph_pyg: common: - output_types: [conda, pyproject] @@ -426,32 +430,14 @@ dependencies: packages: - *dask - *dask_cuda - - *distributed - *numba - *numpy - *thrift - *ucx_py - - output_types: conda - packages: - - *dask-core_conda - output_types: pyproject packages: - *cugraph - - cugraph-service-client==23.10.* - doc: - common: - - output_types: [conda] - packages: - - doxygen - - nbsphinx - - numpydoc - - pydata-sphinx-theme - - recommonmark - - sphinx - - sphinxcontrib-websupport - - sphinx-markdown-tables - - sphinx-copybutton - - pylibcugraphops==23.10.* + - cugraph-service-client==23.12.* test_cpp: common: - output_types: conda @@ -486,7 +472,7 @@ dependencies: - scikit-learn>=0.23.1 - output_types: [conda] packages: - - pylibwholegraph==23.10.* + - pylibwholegraph==23.12.* test_python_pylibcugraph: common: - output_types: [conda, pyproject] @@ -503,8 +489,7 @@ dependencies: common: - output_types: [conda] packages: - - cugraph==23.10.* - - pylibcugraphops==23.10.* + - cugraph==23.12.* - pytorch>=2.0 - pytorch-cuda==11.8 - dgl>=1.1.0.cu* @@ -512,17 +497,16 @@ dependencies: common: - output_types: [conda] packages: - - cugraph==23.10.* - - pylibcugraphops==23.10.* - - pytorch==2.0 + - cugraph==23.12.* + - pytorch>=2.0 - pytorch-cuda==11.8 - - pyg=2.3.1=*torch_2.0.0*cu118* + - pyg>=2.4.0 depends_on_rmm: common: - output_types: conda packages: - - &rmm_conda rmm==23.10.* + - &rmm_conda rmm==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -532,12 +516,12 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &rmm_packages_pip_cu12 - - rmm-cu12==23.10.* + - rmm-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *rmm_packages_pip_cu12} - {matrix: {cuda: "12.0"}, packages: *rmm_packages_pip_cu12} - matrix: {cuda: "11.8"} packages: &rmm_packages_pip_cu11 - - rmm-cu11==23.10.* + - rmm-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *rmm_packages_pip_cu11} - {matrix: {cuda: "11.4"}, packages: *rmm_packages_pip_cu11} - {matrix: {cuda: "11.2"}, packages: *rmm_packages_pip_cu11} @@ -547,7 +531,7 @@ dependencies: common: - output_types: conda packages: - - &cudf_conda cudf==23.10.* + - &cudf_conda cudf==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -557,12 +541,12 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &cudf_packages_pip_cu12 - - cudf-cu12==23.10.* + - cudf-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *cudf_packages_pip_cu12} - {matrix: {cuda: "12.0"}, packages: *cudf_packages_pip_cu12} - matrix: {cuda: "11.8"} packages: &cudf_packages_pip_cu11 - - cudf-cu11==23.10.* + - cudf-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *cudf_packages_pip_cu11} - {matrix: {cuda: "11.4"}, packages: *cudf_packages_pip_cu11} - {matrix: {cuda: "11.2"}, packages: *cudf_packages_pip_cu11} @@ -572,7 +556,7 @@ dependencies: common: - output_types: conda packages: - - &dask_cudf_conda dask-cudf==23.10.* + - &dask_cudf_conda dask-cudf==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -582,12 +566,12 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &dask_cudf_packages_pip_cu12 - - dask-cudf-cu12==23.10.* + - dask-cudf-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *dask_cudf_packages_pip_cu12} - {matrix: {cuda: "12.0"}, packages: *dask_cudf_packages_pip_cu12} - matrix: {cuda: "11.8"} packages: &dask_cudf_packages_pip_cu11 - - dask-cudf-cu11==23.10.* + - dask-cudf-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *dask_cudf_packages_pip_cu11} - {matrix: {cuda: "11.4"}, packages: *dask_cudf_packages_pip_cu11} - {matrix: {cuda: "11.2"}, packages: *dask_cudf_packages_pip_cu11} @@ -597,7 +581,7 @@ dependencies: common: - output_types: conda packages: - - &pylibraft_conda pylibraft==23.10.* + - &pylibraft_conda pylibraft==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -607,12 +591,12 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &pylibraft_packages_pip_cu12 - - pylibraft-cu12==23.10.* + - pylibraft-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *pylibraft_packages_pip_cu12} - {matrix: {cuda: "12.0"}, packages: *pylibraft_packages_pip_cu12} - matrix: {cuda: "11.8"} packages: &pylibraft_packages_pip_cu11 - - pylibraft-cu11==23.10.* + - pylibraft-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *pylibraft_packages_pip_cu11} - {matrix: {cuda: "11.4"}, packages: *pylibraft_packages_pip_cu11} - {matrix: {cuda: "11.2"}, packages: *pylibraft_packages_pip_cu11} @@ -622,7 +606,7 @@ dependencies: common: - output_types: conda packages: - - &raft_dask_conda raft-dask==23.10.* + - &raft_dask_conda raft-dask==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -632,12 +616,12 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &raft_dask_packages_pip_cu12 - - raft-dask-cu12==23.10.* + - raft-dask-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *raft_dask_packages_pip_cu12} - {matrix: {cuda: "12.0"}, packages: *raft_dask_packages_pip_cu12} - matrix: {cuda: "11.8"} packages: &raft_dask_packages_pip_cu11 - - raft-dask-cu11==23.10.* + - raft-dask-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *raft_dask_packages_pip_cu11} - {matrix: {cuda: "11.4"}, packages: *raft_dask_packages_pip_cu11} - {matrix: {cuda: "11.2"}, packages: *raft_dask_packages_pip_cu11} @@ -647,7 +631,7 @@ dependencies: common: - output_types: conda packages: - - &pylibcugraph_conda pylibcugraph==23.10.* + - &pylibcugraph_conda pylibcugraph==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -657,17 +641,42 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &pylibcugraph_packages_pip_cu12 - - pylibcugraph-cu12==23.10.* + - pylibcugraph-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *pylibcugraph_packages_pip_cu12} - {matrix: {cuda: "12.0"}, packages: *pylibcugraph_packages_pip_cu12} - matrix: {cuda: "11.8"} packages: &pylibcugraph_packages_pip_cu11 - - pylibcugraph-cu11==23.10.* + - pylibcugraph-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *pylibcugraph_packages_pip_cu11} - {matrix: {cuda: "11.4"}, packages: *pylibcugraph_packages_pip_cu11} - {matrix: {cuda: "11.2"}, packages: *pylibcugraph_packages_pip_cu11} - {matrix: null, packages: [*pylibcugraph_conda]} + depends_on_pylibcugraphops: + common: + - output_types: conda + packages: + - &pylibcugraphops_conda pylibcugraphops==23.12.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &pylibcugraphops_packages_pip_cu12 + - pylibcugraphops-cu12==23.12.* + - {matrix: {cuda: "12.1"}, packages: *pylibcugraphops_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *pylibcugraphops_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &pylibcugraphops_packages_pip_cu11 + - pylibcugraphops-cu11==23.12.* + - {matrix: {cuda: "11.5"}, packages: *pylibcugraphops_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *pylibcugraphops_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *pylibcugraphops_packages_pip_cu11} + - {matrix: null, packages: [*pylibcugraphops_conda]} + depends_on_cupy: common: - output_types: conda diff --git a/docs/cugraph/Makefile b/docs/cugraph/Makefile index 32237aa2cc0..f92d0be6910 100644 --- a/docs/cugraph/Makefile +++ b/docs/cugraph/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = "-v" SPHINXBUILD = sphinx-build SPHINXPROJ = cugraph SOURCEDIR = source diff --git a/docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst deleted file mode 100644 index e172309fae2..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst +++ /dev/null @@ -1,16 +0,0 @@ -============================= -Operators on Bipartite Graphs -============================= - -.. currentmodule:: pylibcugraphops - -Update Edges: Concatenation or Sum of Edge and Node Features ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.update_efeat_bipartite_e2e_concat_fwd - operators.update_efeat_bipartite_e2e_concat_bwd - - operators.update_efeat_bipartite_e2e_sum_fwd - operators.update_efeat_bipartite_e2e_sum_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst b/docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst new file mode 100644 index 00000000000..5545bebe975 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst @@ -0,0 +1,3 @@ +cugraph-ops C++ API Reference +============================= + diff --git a/docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst deleted file mode 100644 index 387844f684a..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst +++ /dev/null @@ -1,83 +0,0 @@ -======================== -Operators on Full Graphs -======================== - -.. currentmodule:: pylibcugraphops - -Simple Neighborhood Aggregator (SAGEConv) ------------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_simple_fg_n2n_fwd - operators.agg_simple_fg_n2n_bwd - operators.agg_simple_fg_e2n_fwd - operators.agg_simple_fg_e2n_bwd - operators.agg_simple_fg_n2n_e2n_fwd - operators.agg_simple_fg_n2n_e2n_bwd - - operators.agg_concat_fg_n2n_fwd - operators.agg_concat_fg_n2n_bwd - operators.agg_concat_fg_e2n_fwd - operators.agg_concat_fg_e2n_bwd - operators.agg_concat_fg_n2n_e2n_fwd - operators.agg_concat_fg_n2n_e2n_bwd - -Weighted Neighborhood Aggregation ---------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_weighted_fg_n2n_fwd - operators.agg_weighted_fg_n2n_bwd - operators.agg_concat_weighted_fg_n2n_fwd - operators.agg_concat_weighted_fg_n2n_bwd - -Heterogenous Aggregator using Basis Decomposition (RGCNConv) ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_hg_basis_fg_n2n_post_fwd - operators.agg_hg_basis_fg_n2n_post_bwd - -Graph Attention (GATConv/GATv2Conv) ------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.mha_gat_fg_n2n_fwd - operators.mha_gat_fg_n2n_bwd - operators.mha_gat_fg_n2n_efeat_fwd - operators.mha_gat_fg_n2n_efeat_bwd - - operators.mha_gat_v2_fg_n2n_fwd - operators.mha_gat_v2_fg_n2n_bwd - operators.mha_gat_v2_fg_n2n_efeat_fwd - operators.mha_gat_v2_fg_n2n_efeat_bwd - -Transformer-like Graph Attention (TransformerConv) --------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.mha_gat_v2_fg_n2n_fwd - operators.mha_gat_v2_fg_n2n_bwd - operators.mha_gat_v2_fg_n2n_efeat_fwd - operators.mha_gat_v2_fg_n2n_efeat_bwd - -Directional Message-Passing (DMPNN) ------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_dmpnn_fg_e2e_fwd - operators.agg_dmpnn_fg_e2e_bwd - -Graph Pooling -------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.pool_fg_n2s_fwd - operators.pool_fg_n2s_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst b/docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst deleted file mode 100644 index 9289ce53e39..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst +++ /dev/null @@ -1,33 +0,0 @@ -=========== -Graph types -=========== - -.. currentmodule:: pylibcugraphops - -Message-Flow Graph (MFG) -------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - make_mfg_csr - -Heterogenous MFG ----------------- -.. autosummary:: - :toctree: ../api/ops/ - - make_mfg_csr_hg - -"Full" Graph (FG) ------------------ -.. autosummary:: - :toctree: ../api/ops/ - - make_fg_csr - -Heterogenous FG ---------------- -.. autosummary:: - :toctree: ../api/ops/ - - make_fg_csr_hg diff --git a/docs/cugraph/source/api_docs/cugraph-ops/index.rst b/docs/cugraph/source/api_docs/cugraph-ops/index.rst index e2338dc1833..fdfd5baab96 100644 --- a/docs/cugraph/source/api_docs/cugraph-ops/index.rst +++ b/docs/cugraph/source/api_docs/cugraph-ops/index.rst @@ -1,4 +1,3 @@ -========================= cugraph-ops API reference ========================= @@ -8,11 +7,5 @@ This page provides a list of all publicly accessible modules, methods and classe :maxdepth: 2 :caption: API Documentation - graph_types - pytorch - mfg_operators - bipartite_operators - static_operators - fg_operators - dimenet - pytorch + python/index + c_cpp/index \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst deleted file mode 100644 index f3dd1faa245..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst +++ /dev/null @@ -1,31 +0,0 @@ -================================ -Operators on Message-Flow Graphs -================================ - -.. currentmodule:: pylibcugraphops - -Simple Neighborhood Aggregator (SAGEConv) ------------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_simple_mfg_n2n_fwd - operators.agg_simple_mfg_n2n_bwd - operators.agg_concat_mfg_n2n_fwd - operators.agg_concat_mfg_n2n_bwd - -Graph Attention (GATConv) -------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.mha_gat_mfg_n2n_fwd - operators.mha_gat_mfg_n2n_bwd - -Heterogenous Aggregator using Basis Decomposition (RGCNConv) ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_hg_basis_mfg_n2n_post_fwd - operators.agg_hg_basis_mfg_n2n_post_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/dimenet.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/dimenet.rst similarity index 89% rename from docs/cugraph/source/api_docs/cugraph-ops/dimenet.rst rename to docs/cugraph/source/api_docs/cugraph-ops/python/dimenet.rst index b709464c7e6..6fadcc57b22 100644 --- a/docs/cugraph/source/api_docs/cugraph-ops/dimenet.rst +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/dimenet.rst @@ -7,7 +7,7 @@ Dimenet operators Radial Basis Functions ---------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops dimenet.radial_basis_fwd dimenet.radial_basis_bwd @@ -16,7 +16,7 @@ Radial Basis Functions Edge-to-Edge Aggregation ------------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops dimenet.agg_edge_to_edge_fwd dimenet.agg_edge_to_edge_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst new file mode 100644 index 00000000000..141d40393a5 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst @@ -0,0 +1,34 @@ +=========== +Graph types +=========== + +.. currentmodule:: pylibcugraphops + + +CSC Graph +----------------- +.. autosummary:: + :toctree: ../../api/ops + + make_csc + +Heterogenous CSC Graph +---------------------- +.. autosummary:: + :toctree: ../../api/ops + + make_csc_hg + +Bipartite Graph +----------------- +.. autosummary:: + :toctree: ../../api/ops + + make_bipartite_csc + +Heterogenous Bipartite Graph +---------------------------- +.. autosummary:: + :toctree: ../../api/ops + + make_bipartite_csc_hg diff --git a/docs/cugraph/source/api_docs/cugraph-ops/python/index.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/index.rst new file mode 100644 index 00000000000..082c7741f23 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/index.rst @@ -0,0 +1,13 @@ +cugraph-ops Python API reference +================================ + +This page provides a list of all publicly accessible modules, methods and classes through `pylibcugraphops.*` namespace. + +.. toctree:: + :maxdepth: 2 + :caption: API Documentation + + graph_types + operators + dimenet + pytorch diff --git a/docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst new file mode 100644 index 00000000000..3e6664b2db5 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst @@ -0,0 +1,93 @@ +============================= +Operators for Message-Passing +============================= + +.. currentmodule:: pylibcugraphops + +Simple Neighborhood Aggregator (SAGEConv) +----------------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_simple_n2n_fwd + operators.agg_simple_n2n_bwd + operators.agg_simple_e2n_fwd + operators.agg_simple_e2n_bwd + operators.agg_simple_n2n_e2n_fwd + operators.agg_simple_n2n_e2n_bwd + + operators.agg_concat_n2n_fwd + operators.agg_concat_n2n_bwd + operators.agg_concat_e2n_fwd + operators.agg_concat_e2n_bwd + operators.agg_concat_n2n_e2n_fwd + operators.agg_concat_n2n_e2n_bwd + + +Weighted Neighborhood Aggregation +--------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_weighted_n2n_fwd + operators.agg_weighted_n2n_bwd + operators.agg_concat_weighted_n2n_fwd + operators.agg_concat_weighted_n2n_bwd + +Heterogenous Aggregator using Basis Decomposition (RGCNConv) +------------------------------------------------------------ +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_hg_basis_n2n_post_fwd + operators.agg_hg_basis_n2n_post_bwd + +Graph Attention (GATConv/GATv2Conv) +----------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.mha_gat_n2n_fwd + operators.mha_gat_n2n_bwd + operators.mha_gat_n2n_efeat_fwd + operators.mha_gat_n2n_efeat_bwd + + operators.mha_gat_v2_n2n_fwd + operators.mha_gat_v2_n2n_bwd + operators.mha_gat_v2_n2n_efeat_fwd + operators.mha_gat_v2_n2n_efeat_bwd + +Transformer-like Graph Attention (TransformerConv) +-------------------------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.mha_gat_v2_n2n_fwd + operators.mha_gat_v2_n2n_bwd + operators.mha_gat_v2_n2n_efeat_fwd + operators.mha_gat_v2_n2n_efeat_bwd + +Directional Message-Passing (DMPNN) +----------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_dmpnn_e2e_fwd + operators.agg_dmpnn_e2e_bwd + +Update Edges: Concatenation or Sum of Edge and Node Features +------------------------------------------------------------ +.. autosummary:: + :toctree: ../../api/ops + + operators.update_efeat_e2e_concat_fwd + operators.update_efeat_e2e_concat_bwd + + operators.update_efeat_e2e_sum_fwd + operators.update_efeat_e2e_sum_bwd + + operators.update_efeat_e2e_concat_fwd + operators.update_efeat_e2e_concat_bwd + + operators.update_efeat_e2e_sum_fwd + operators.update_efeat_e2e_sum_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/pytorch.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/pytorch.rst similarity index 59% rename from docs/cugraph/source/api_docs/cugraph-ops/pytorch.rst rename to docs/cugraph/source/api_docs/cugraph-ops/python/pytorch.rst index 83800fbc546..d2074df15b0 100644 --- a/docs/cugraph/source/api_docs/cugraph-ops/pytorch.rst +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/pytorch.rst @@ -2,35 +2,35 @@ PyTorch Autograd Wrappers ========================== -.. currentmodule:: pylibcugraphops +.. currentmodule:: pylibcugraphops.pytorch Simple Neighborhood Aggregator (SAGEConv) ----------------------------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.agg_concat_n2n + operators.agg_concat_n2n Graph Attention (GATConv/GATv2Conv) ----------------------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.mha_gat_n2n - pytorch.operators.mha_gat_v2_n2n + operators.mha_gat_n2n + operators.mha_gat_v2_n2n Heterogenous Aggregator using Basis Decomposition (RGCNConv) ------------------------------------------------------------ .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.agg_hg_basis_n2n_post + operators.agg_hg_basis_n2n_post Update Edges: Concatenation or Sum of Edge and Node Features ------------------------------------------------------------ .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.update_efeat_bipartite_e2e - pytorch.operators.update_efeat_static_e2e + operators.update_efeat_e2e + operators.update_efeat_e2e diff --git a/docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst deleted file mode 100644 index f3ecc068f22..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst +++ /dev/null @@ -1,16 +0,0 @@ -========================== -Operators on Static Graphs -========================== - -.. currentmodule:: pylibcugraphops - -Update Edges: Concatenation or Sum of Edge and Node Features ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.update_efeat_static_e2e_concat_fwd - operators.update_efeat_static_e2e_concat_bwd - - operators.update_efeat_static_e2e_sum_fwd - operators.update_efeat_static_e2e_sum_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst b/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst index 2cd8969aa66..f7d7f5f2262 100644 --- a/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst +++ b/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst @@ -9,6 +9,6 @@ cugraph-pyg .. autosummary:: :toctree: ../api/cugraph-pyg/ - cugraph_pyg.data.cugraph_store.EXPERIMENTAL__CuGraphStore - cugraph_pyg.sampler.cugraph_sampler.EXPERIMENTAL__CuGraphSampler +.. cugraph_pyg.data.cugraph_store.EXPERIMENTAL__CuGraphStore +.. cugraph_pyg.sampler.cugraph_sampler.EXPERIMENTAL__CuGraphSampler diff --git a/docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst b/docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst deleted file mode 100644 index 34b812785d3..00000000000 --- a/docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst +++ /dev/null @@ -1,4 +0,0 @@ -CuGraph C and C++ API Links -=========================== - -coming soon - see https://docs.rapids.ai/api/libcugraph/nightly/ \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph_c/centrality.rst b/docs/cugraph/source/api_docs/cugraph_c/centrality.rst new file mode 100644 index 00000000000..f34e26ad76e --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/centrality.rst @@ -0,0 +1,51 @@ +Centrality +========== + +PageRank +-------- +.. doxygenfunction:: cugraph_pagerank + :project: libcugraph + +.. doxygenfunction:: cugraph_pagerank_allow_nonconvergence + :project: libcugraph + +Personalized PageRank +--------------------- +.. doxygenfunction:: cugraph_personalized_pagerank + :project: libcugraph + +.. doxygenfunction:: cugraph_personalized_pagerank_allow_nonconvergence + :project: libcugraph + +Eigenvector Centrality +---------------------- +.. doxygenfunction:: cugraph_eigenvector_centrality + :project: libcugraph + +Katz Centrality +--------------- +.. doxygenfunction:: cugraph_katz_centrality + :project: libcugraph + +Betweenness Centrality +---------------------- +.. doxygenfunction:: cugraph_betweenness_centrality + :project: libcugraph + +Edge Betweenness Centrality +--------------------------- +.. doxygenfunction:: cugraph_edge_betweenness_centrality + :project: libcugraph + +HITS Centrality +--------------- +.. doxygenfunction:: cugraph_hits + :project: libcugraph + +Centrality Support Functions +---------------------------- + .. doxygengroup:: centrality + :project: libcugraph + :members: + :content-only: + diff --git a/docs/cugraph/source/api_docs/cugraph_c/community.rst b/docs/cugraph/source/api_docs/cugraph_c/community.rst new file mode 100644 index 00000000000..0bbfe365c4d --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/community.rst @@ -0,0 +1,63 @@ +Community +========= + +.. role:: py(code) + :language: c + :class: highlight + +``#include `` + +Triangle Counting +----------------- +.. doxygenfunction:: cugraph_triangle_count + :project: libcugraph + +Louvain +------- +.. doxygenfunction:: cugraph_louvain + :project: libcugraph + +Leiden +------ +.. doxygenfunction:: cugraph_leiden + :project: libcugraph + +ECG +--- +.. doxygenfunction:: cugraph_ecg + :project: libcugraph + +Extract Egonet +-------------- +.. doxygenfunction:: cugraph_extract_ego + :project: libcugraph + +Balanced Cut +------------ +.. doxygenfunction:: cugraph_balanced_cut_clustering + :project: libcugraph + +Spectral Clustering - Modularity Maximization +--------------------------------------------- +.. doxygenfunction:: cugraph_spectral_modularity_maximization + :project: libcugraph + +.. doxygenfunction:: cugraph_analyze_clustering_modularity + :project: libcugraph + +Spectral Clusteriong - Edge Cut +------------------------------- +.. doxygenfunction:: cugraph_analyze_clustering_edge_cut + :project: libcugraph + +.. doxygenfunction:: cugraph_analyze_clustering_ratio_cut + :project: libcugraph + + +Community Support Functions +--------------------------- + .. doxygengroup:: community + :project: libcugraph + :members: + :content-only: + diff --git a/docs/cugraph/source/api_docs/cugraph_c/core.rst b/docs/cugraph/source/api_docs/cugraph_c/core.rst new file mode 100644 index 00000000000..34456c65e43 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/core.rst @@ -0,0 +1,21 @@ +Core +==== + + +Core Number +----------- +.. doxygenfunction:: cugraph_core_number + :project: libcugraph + +K-Core +------ +.. doxygenfunction:: cugraph_k_core + :project: libcugraph + + +Core Support Functions +---------------------- + .. doxygengroup:: core + :project: libcugraph + :members: + :content-only: diff --git a/docs/cugraph/source/api_docs/cugraph_c/index.rst b/docs/cugraph/source/api_docs/cugraph_c/index.rst new file mode 100644 index 00000000000..3dd37dbc374 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/index.rst @@ -0,0 +1,16 @@ +=========================== +cuGraph C API documentation +=========================== + + +.. toctree:: + :maxdepth: 3 + :caption: API Documentation + + centrality.rst + community.rst + core.rst + labeling.rst + sampling.rst + similarity.rst + traversal.rst diff --git a/docs/cugraph/source/api_docs/cugraph_c/labeling.rst b/docs/cugraph/source/api_docs/cugraph_c/labeling.rst new file mode 100644 index 00000000000..af105ee8fc9 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/labeling.rst @@ -0,0 +1,20 @@ +Components +========== + + +Weakly Connected Components +--------------------------- +.. doxygenfunction:: cugraph_weakly_connected_components + :project: libcugraph + +Strongly Connected Components +----------------------------- +.. doxygenfunction:: cugraph_strongly_connected_components + :project: libcugraph + +Support +------- + .. doxygengroup:: labeling + :project: libcugraph + :members: + :content-only: \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph_c/sampling.rst b/docs/cugraph/source/api_docs/cugraph_c/sampling.rst new file mode 100644 index 00000000000..21b837daf93 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/sampling.rst @@ -0,0 +1,37 @@ +Sampling +======== + +Uniform Random Walks +-------------------- +.. doxygenfunction:: cugraph_uniform_random_walks + :project: libcugraph + +Biased Random Walks +-------------------- +.. doxygenfunction:: cugraph_biased_random_walks + :project: libcugraph + +Random Walks via Node2Vec +------------------------- +.. doxygenfunction:: cugraph_node2vec_random_walks + :project: libcugraph + +Node2Vec +-------- +.. doxygenfunction:: cugraph_node2vec + :project: libcugraph + +Uniform Neighborhood Sampling +----------------------------- +.. doxygenfunction:: cugraph_uniform_neighbor_sample_with_edge_properties + :project: libcugraph + +.. doxygenfunction:: cugraph_uniform_neighbor_sample + :project: libcugraph + +Support +------- +.. doxygengroup:: samplingC + :project: libcugraph + :members: + :content-only: diff --git a/docs/cugraph/source/api_docs/cugraph_c/similarity.rst b/docs/cugraph/source/api_docs/cugraph_c/similarity.rst new file mode 100644 index 00000000000..fba07ad206c --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/similarity.rst @@ -0,0 +1,25 @@ +Similarity +========== + + +Jaccard +------- +.. doxygenfunction:: cugraph_jaccard_coefficients + :project: libcugraph + +Sorensen +-------- +.. doxygenfunction:: cugraph_sorensen_coefficients + :project: libcugraph + +Overlap +------- +.. doxygenfunction:: cugraph_overlap_coefficients + :project: libcugraph + +Support +------- +.. doxygengroup:: similarity + :project: libcugraph + :members: + :content-only: \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph_c/traversal.rst b/docs/cugraph/source/api_docs/cugraph_c/traversal.rst new file mode 100644 index 00000000000..c90760e9e79 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/traversal.rst @@ -0,0 +1,30 @@ +Traversal +========== + + +Breadth First Search (BFS) +-------------------------- +.. doxygenfunction:: cugraph_bfs + :project: libcugraph + +Single-Source Shortest-Path (SSSP) +---------------------------------- +.. doxygenfunction:: cugraph_sssp + :project: libcugraph + +Path Extraction +--------------- +.. doxygenfunction:: cugraph_extract_paths + :project: libcugraph + +Extract Max Path Length +----------------------- +.. doxygenfunction:: cugraph_extract_paths_result_get_max_path_length + :project: libcugraph + +Support +------- +.. doxygengroup:: traversal + :project: libcugraph + :members: + :content-only: \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/index.rst b/docs/cugraph/source/api_docs/index.rst index 45f7210f5a2..74ca98bb98d 100644 --- a/docs/cugraph/source/api_docs/index.rst +++ b/docs/cugraph/source/api_docs/index.rst @@ -1,16 +1,39 @@ -Python API reference -==================== +API Reference +============= This page provides a list of all publicly accessible Python modules with in the Graph collection +Core Graph API Documentation +---------------------------- + .. toctree:: - :maxdepth: 2 - :caption: Python API Documentation + :maxdepth: 3 + :caption: Core Graph API Documentation cugraph/index.rst plc/pylibcugraph.rst + cugraph_c/index.rst + cugraph_cpp/index.rst + +Graph Nerual Networks API Documentation +--------------------------------------- + +.. toctree:: + :maxdepth: 3 + :caption: Graph Nerual Networks API Documentation + cugraph-dgl/cugraph_dgl.rst cugraph-pyg/cugraph_pyg.rst - service/index.rst cugraph-ops/index.rst + wholegraph/index.rst + +Additional Graph Packages API Documentation +---------------------------------- + +.. toctree:: + :maxdepth: 3 + :caption: Additional Graph Packages API Documentation + + service/index.rst + diff --git a/docs/cugraph/source/api_docs/service/cugraph_service_client.rst b/docs/cugraph/source/api_docs/service/cugraph_service_client.rst index 383b31d269a..7e344d326f7 100644 --- a/docs/cugraph/source/api_docs/service/cugraph_service_client.rst +++ b/docs/cugraph/source/api_docs/service/cugraph_service_client.rst @@ -9,7 +9,7 @@ cugraph-service .. autosummary:: :toctree: ../api/service/ - cugraph_service_client.client.RunAsyncioThread +.. cugraph_service_client.client.RunAsyncioThread cugraph_service_client.client.run_async cugraph_service_client.client.DeviceArrayAllocator cugraph_service_client.client.CugraphServiceClient diff --git a/docs/cugraph/source/api_docs/service/cugraph_service_server.rst b/docs/cugraph/source/api_docs/service/cugraph_service_server.rst index a7e8b547573..09ca8360b6c 100644 --- a/docs/cugraph/source/api_docs/service/cugraph_service_server.rst +++ b/docs/cugraph/source/api_docs/service/cugraph_service_server.rst @@ -9,6 +9,6 @@ cugraph-service .. autosummary:: :toctree: ../api/service/ - cugraph_service_server.cugraph_handler.call_algo +.. cugraph_service_server.cugraph_handler.call_algo cugraph_service_server.cugraph_handler.ExtensionServerFacade cugraph_service_server.cugraph_handler.CugraphHandler diff --git a/docs/cugraph/source/api_docs/wholegraph/index.rst b/docs/cugraph/source/api_docs/wholegraph/index.rst new file mode 100644 index 00000000000..80e231d4610 --- /dev/null +++ b/docs/cugraph/source/api_docs/wholegraph/index.rst @@ -0,0 +1,11 @@ +WholeGraph API reference +======================== + +This page provides WholeGraph API reference + +.. toctree:: + :maxdepth: 2 + :caption: WholeGraph API Documentation + + libwholegraph/index.rst + pylibwholegraph/index.rst diff --git a/docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst b/docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst new file mode 100644 index 00000000000..4ef68abef2d --- /dev/null +++ b/docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst @@ -0,0 +1,228 @@ +===================== +libwholegraph API doc +===================== + +Doxygen WholeGraph C API documentation +-------------------------------------- +For doxygen documentation, please refer to `Doxygen Documentation <../../doxygen_docs/libwholegraph/html/index.html>`_ + +WholeGraph C API documentation +------------------------------ + +Library Level APIs +++++++++++++++++++ + +.. doxygenenum:: wholememory_error_code_t + :project: libwholegraph +.. doxygenfunction:: wholememory_init + :project: libwholegraph +.. doxygenfunction:: wholememory_finalize + :project: libwholegraph +.. doxygenfunction:: fork_get_device_count + :project: libwholegraph + +WholeMemory Communicator APIs ++++++++++++++++++++++++++++++ + +.. doxygentypedef:: wholememory_comm_t + :project: libwholegraph +.. doxygenstruct:: wholememory_unique_id_t + :project: libwholegraph +.. doxygenfunction:: wholememory_create_unique_id + :project: libwholegraph +.. doxygenfunction:: wholememory_create_communicator + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_communicator + :project: libwholegraph +.. doxygenfunction:: wholememory_communicator_get_rank + :project: libwholegraph +.. doxygenfunction:: wholememory_communicator_get_size + :project: libwholegraph +.. doxygenfunction:: wholememory_communicator_barrier + :project: libwholegraph + +WholeMemoryHandle APIs +++++++++++++++++++++++ + +.. doxygenenum:: wholememory_memory_type_t + :project: libwholegraph +.. doxygenenum:: wholememory_memory_location_t + :project: libwholegraph +.. doxygentypedef:: wholememory_handle_t + :project: libwholegraph +.. doxygenstruct:: wholememory_gref_t + :project: libwholegraph +.. doxygenfunction:: wholememory_malloc + :project: libwholegraph +.. doxygenfunction:: wholememory_free + :project: libwholegraph +.. doxygenfunction:: wholememory_get_communicator + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_type + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_location + :project: libwholegraph +.. doxygenfunction:: wholememory_get_total_size + :project: libwholegraph +.. doxygenfunction:: wholememory_get_data_granularity + :project: libwholegraph +.. doxygenfunction:: wholememory_get_local_memory + :project: libwholegraph +.. doxygenfunction:: wholememory_get_rank_memory + :project: libwholegraph +.. doxygenfunction:: wholememory_get_global_pointer + :project: libwholegraph +.. doxygenfunction:: wholememory_get_global_reference + :project: libwholegraph +.. doxygenfunction:: wholememory_determine_partition_plan + :project: libwholegraph +.. doxygenfunction:: wholememory_determine_entry_partition_plan + :project: libwholegraph +.. doxygenfunction:: wholememory_get_partition_plan + :project: libwholegraph +.. doxygenfunction:: wholememory_load_from_file + :project: libwholegraph +.. doxygenfunction:: wholememory_store_to_file + :project: libwholegraph + +WholeMemoryTensor APIs +++++++++++++++++++++++ + +.. doxygenenum:: wholememory_dtype_t + :project: libwholegraph +.. doxygenstruct:: wholememory_array_description_t + :project: libwholegraph +.. doxygenstruct:: wholememory_matrix_description_t + :project: libwholegraph +.. doxygenstruct:: wholememory_tensor_description_t + :project: libwholegraph +.. doxygentypedef:: wholememory_tensor_t + :project: libwholegraph +.. doxygenfunction:: wholememory_dtype_get_element_size + :project: libwholegraph +.. doxygenfunction:: wholememory_dtype_is_floating_number + :project: libwholegraph +.. doxygenfunction:: wholememory_dtype_is_integer_number + :project: libwholegraph +.. doxygenfunction:: wholememory_create_array_desc + :project: libwholegraph +.. doxygenfunction:: wholememory_create_matrix_desc + :project: libwholegraph +.. doxygenfunction:: wholememory_initialize_tensor_desc + :project: libwholegraph +.. doxygenfunction:: wholememory_copy_array_desc_to_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_copy_array_desc_to_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_copy_matrix_desc_to_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_convert_tensor_desc_to_array + :project: libwholegraph +.. doxygenfunction:: wholememory_convert_tensor_desc_to_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_element_count_from_array + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_size_from_array + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_element_count_from_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_size_from_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_element_count_from_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_size_from_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_unsqueeze_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_create_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_make_tensor_from_pointer + :project: libwholegraph +.. doxygenfunction:: wholememory_make_tensor_from_handle + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_has_handle + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_memory_handle + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_tensor_description + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_global_reference + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_map_local_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_data_pointer + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_entry_per_partition + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_subtensor + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_root + :project: libwholegraph + +Ops on WholeMemory Tensors +++++++++++++++++++++++++++ + +.. doxygenfunction:: wholememory_gather + :project: libwholegraph +.. doxygenfunction:: wholememory_scatter + :project: libwholegraph + +WholeTensorEmbedding APIs ++++++++++++++++++++++++++ + +.. doxygentypedef:: wholememory_embedding_cache_policy_t + :project: libwholegraph +.. doxygentypedef:: wholememory_embedding_optimizer_t + :project: libwholegraph +.. doxygentypedef:: wholememory_embedding_t + :project: libwholegraph +.. doxygenenum:: wholememory_access_type_t + :project: libwholegraph +.. doxygenenum:: wholememory_optimizer_type_t + :project: libwholegraph +.. doxygenfunction:: wholememory_create_embedding_optimizer + :project: libwholegraph +.. doxygenfunction:: wholememory_optimizer_set_parameter + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_embedding_optimizer + :project: libwholegraph +.. doxygenfunction:: wholememory_create_embedding_cache_policy + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_embedding_cache_policy + :project: libwholegraph +.. doxygenfunction:: wholememory_create_embedding + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_embedding + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_get_embedding_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_gather + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_gather_gradient_apply + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_get_optimizer_state_names + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_get_optimizer_state + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_writeback_cache + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_drop_all_cache + :project: libwholegraph + +Ops on graphs stored in WholeMemory ++++++++++++++++++++++++++++++++++++ + +.. doxygenfunction:: wholegraph_csr_unweighted_sample_without_replacement + :project: libwholegraph +.. doxygenfunction:: wholegraph_csr_weighted_sample_without_replacement + :project: libwholegraph + +Miscellaneous Ops for graph ++++++++++++++++++++++++++++ + +.. doxygenfunction:: graph_append_unique + :project: libwholegraph +.. doxygenfunction:: csr_add_self_loop + :project: libwholegraph diff --git a/docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst b/docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst new file mode 100644 index 00000000000..67aab00acef --- /dev/null +++ b/docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst @@ -0,0 +1,38 @@ +======================= +pylibwholegraph API doc +======================= + +.. currentmodule:: pylibwholegraph + +APIs +---- +.. autosummary:: + :toctree: ../../api/wg + + torch.initialize.init_torch_env + torch.initialize.init_torch_env_and_create_wm_comm + torch.initialize.finalize + torch.comm.WholeMemoryCommunicator + torch.comm.set_world_info + torch.comm.create_group_communicator + torch.comm.destroy_communicator + torch.comm.get_global_communicator + torch.comm.get_local_node_communicator + torch.comm.get_local_device_communicator + torch.tensor.WholeMemoryTensor + torch.tensor.create_wholememory_tensor + torch.tensor.create_wholememory_tensor_from_filelist + torch.tensor.destroy_wholememory_tensor + torch.embedding.WholeMemoryOptimizer + torch.embedding.create_wholememory_optimizer + torch.embedding.destroy_wholememory_optimizer + torch.embedding.WholeMemoryCachePolicy + torch.embedding.create_wholememory_cache_policy + torch.embedding.create_builtin_cache_policy + torch.embedding.destroy_wholememory_cache_policy + torch.embedding.WholeMemoryEmbedding + torch.embedding.create_embedding + torch.embedding.create_embedding_from_filelist + torch.embedding.destroy_embedding + torch.embedding.WholeMemoryEmbeddingModule + torch.graph_structure.GraphStructure diff --git a/docs/cugraph/source/basics/cugraph_intro.md b/docs/cugraph/source/basics/cugraph_intro.md index 0684129503f..10d14f8a0d7 100644 --- a/docs/cugraph/source/basics/cugraph_intro.md +++ b/docs/cugraph/source/basics/cugraph_intro.md @@ -21,7 +21,7 @@ call graph algorithms using data stored in a GPU DataFrame, NetworkX Graphs, or CuPy or SciPy sparse Matrix. -# Vision +## Vision The vision of RAPIDS cuGraph is to ___make graph analysis ubiquitous to the point that users just think in terms of analysis and not technologies or frameworks___. This is a goal that many of us on the cuGraph team have been @@ -49,7 +49,7 @@ RAPIDS and DASK allows cuGraph to scale to multiple GPUs to support multi-billion edge graphs. -# Terminology +## Terminology cuGraph is a collection of GPU accelerated graph algorithms and graph utility functions. The application of graph analysis covers a lot of areas. @@ -67,8 +67,7 @@ documentation we will mostly use the terms __Node__ and __Edge__ to better match NetworkX preferred term use, as well as other Python-based tools. At the CUDA/C layer, we favor the mathematical terms of __Vertex__ and __Edge__. -# Roadmap -GitHub does not provide a robust project management interface, and so a roadmap turns into simply a projection of when work will be completed and not a complete picture of everything that needs to be done. To capture the work that requires multiple steps, issues are labels as “EPIC” and include multiple subtasks that could span multiple releases. The EPIC will be in the release where work in expected to be completed. A better roadmap is being worked an image of the roadmap will be posted when ready. - * GitHub Project Board: https://github.com/rapidsai/cugraph/projects/28 + + \ No newline at end of file diff --git a/docs/cugraph/source/conf.py b/docs/cugraph/source/conf.py index a96f0fc1e82..3f7ef7deb03 100644 --- a/docs/cugraph/source/conf.py +++ b/docs/cugraph/source/conf.py @@ -37,6 +37,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + "breathe", "sphinx.ext.intersphinx", "sphinx.ext.autodoc", "sphinx.ext.autosummary", @@ -76,9 +77,9 @@ # built documents. # # The short X.Y version. -version = '23.10' +version = '23.12' # The full version, including alpha/beta/rc tags. -release = '23.10.00' +release = '23.12.00' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -180,11 +181,10 @@ # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'cugraph', 'cugraph Documentation', - author, 'cugraph', 'One line description of project.', + author, 'cugraph', 'GPU-accelerated graph analysis.', 'Miscellaneous'), ] - # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} @@ -205,4 +205,12 @@ def setup(app): # The following is used by sphinx.ext.linkcode to provide links to github linkcode_resolve = make_linkcode_resolve( "https://github.com/rapidsai/cugraph/blob/{revision}/python/{path}#L{lineno}" -) \ No newline at end of file +) + +breathe_projects = { + 'libcugraph': os.environ['XML_DIR_LIBCUGRAPH'], + 'libcugraphops': os.environ['XML_DIR_LIBCUGRAPHOPS'], + 'libwholegraph': os.environ['XML_DIR_LIBWHOLEGRAPH'] +} + +breathe_default_project = "libcugraph" diff --git a/docs/cugraph/source/graph_support/algorithms.md b/docs/cugraph/source/graph_support/algorithms.md index f6cb7c0d8b1..a1b80e92751 100644 --- a/docs/cugraph/source/graph_support/algorithms.md +++ b/docs/cugraph/source/graph_support/algorithms.md @@ -22,7 +22,7 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | Category | Notebooks | Scale | Notes | | ----------------- | ---------------------------------- | ------------------- | --------------------------------------------------------------- | -| [Centrality](./algorithms/Centrality.md) | [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb) | | | +| [Centrality](./algorithms/Centrality.html ) | [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb) | | | | | [Katz](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Katz.ipynb) | __Multi-GPU__ | | | | [Betweenness Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Betweenness.ipynb) | __Multi-GPU__ | MG as of 23.06 | | | [Edge Betweenness Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Betweenness.ipynb) | __Multi-GPU__ | MG as of 23.08 | @@ -31,12 +31,12 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | Community | | | | | | [Leiden](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Louvain.ipynb) | __Multi-GPU__ | MG as of 23.06 | | | [Louvain](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Louvain.ipynb) | __Multi-GPU__ | | -| | [Ensemble Clustering for Graphs](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ECG.ipynb) | Single-GPU | MG planned for 23.10 | +| | [Ensemble Clustering for Graphs](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ECG.ipynb) | Single-GPU | MG planned for 24.02 | | | [Spectral-Clustering - Balanced Cut](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Spectral-Clustering.ipynb) | Single-GPU | | | | [Spectral-Clustering - Modularity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Spectral-Clustering.ipynb) | Single-GPU | | | | [Subgraph Extraction](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Subgraph-Extraction.ipyn) | Single-GPU | | | | [Triangle Counting](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Triangle-Counting.ipynb) | __Multi-GPU__ | | -| | [K-Truss](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ktruss.ipynb) | Single-GPU | MG planned for 23.10 | +| | [K-Truss](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ktruss.ipynb) | Single-GPU | MG planned for 2024 | | Components | | | | | | [Weakly Connected Components](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/components/ConnectedComponents.ipynb) | __Multi-GPU__ | | | | [Strongly Connected Components](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/components/ConnectedComponents.ipynb) | Single-GPU | | @@ -55,7 +55,7 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | | [Pagerank](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/Pagerank.ipynb) | __Multi-GPU__ | [C++ README](cpp/src/centrality/README.md#Pagerank) | | | [Personal Pagerank]() | __Multi-GPU__ | [C++ README](cpp/src/centrality/README.md#Personalized-Pagerank) | | | [HITS](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/HITS.ipynb) | __Multi-GPU__ | | -| [Link Prediction](./algorithms/Similarity.md) | | | | +| [Link Prediction](algorithms/Similarity.html) | | | | | | [Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) | __Multi-GPU__ | Directed graph only | | | [Weighted Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) | Single-GPU | | | | [Overlap Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Overlap-Similarity.ipynb) | **Multi-GPU** | | @@ -65,8 +65,8 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | | [Uniform Random Walks RW](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/sampling/RandomWalk.ipynb) | __Multi-GPU__ | | | | *Biased Random Walks (RW)* | --- | | | | Egonet | __Multi-GPU__ | | -| | Node2Vec | Single-GPU | | -| | Uniform Neighborhood sampling | __Multi-GPU__ | | +| | Node2Vec | __Multi-GPU__ | | +| | Neighborhood sampling | __Multi-GPU__ | | | Traversal | | | | | | Breadth First Search (BFS) | __Multi-GPU__ | with cutoff support [C++ README](cpp/src/traversal/README.md#BFS) | | | Single Source Shortest Path (SSSP) | __Multi-GPU__ | [C++ README](cpp/src/traversal/README.md#SSSP) | diff --git a/docs/cugraph/source/graph_support/algorithms/Centrality.md b/docs/cugraph/source/graph_support/algorithms/Centrality.md index e42bbe238c6..8119e655236 100644 --- a/docs/cugraph/source/graph_support/algorithms/Centrality.md +++ b/docs/cugraph/source/graph_support/algorithms/Centrality.md @@ -1,7 +1,7 @@ # cuGraph Centrality Notebooks - + The RAPIDS cuGraph Centrality folder contains a collection of Jupyter Notebooks that demonstrate algorithms to identify and quantify the importance of vertices to the structure of the graph. In the diagram above, the highlighted vertices are highly important and are likely answers to questions like: @@ -15,13 +15,15 @@ But which vertices are most important? The answer depends on which measure/algor |Algorithm |Notebooks Containing |Description | | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -|[Degree Centrality](./degree_centrality.md)| [Centrality](./Centrality.ipynb), [Degree](./Degree.ipynb) |Measure based on counting direct connections for each vertex| -|[Betweenness Centrality](./betweenness_centrality.md)| [Centrality](./Centrality.ipynb), [Betweenness](./Betweenness.ipynb) |Number of shortest paths through the vertex| -|[Eigenvector Centrality](./eigenvector_centrality.md)|[Centrality](./Centrality.ipynb), [Eigenvector](./Eigenvector.ipynb)|Measure of connectivity to other important vertices (which also have high connectivity) often referred to as the influence measure of a vertex| -|[Katz Centrality](./katz_centrality.md)|[Centrality](./Centrality.ipynb), [Katz](./Katz.ipynb) |Similar to Eigenvector but has tweaks to measure more weakly connected graph | -|Pagerank|[Centrality](./Centrality.ipynb), [Pagerank](../../link_analysis/Pagerank.ipynb) |Classified as both a link analysis and centrality measure by quantifying incoming links from central vertices. | +|[Degree Centrality](./degree_centrality.md)| [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Degree](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Degree.ipynb) |Measure based on counting direct connections for each vertex| +|[Betweenness Centrality](./betweenness_centrality.md)| [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Betweenness](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Betweenness.ipynb) |Number of shortest paths through the vertex| +|[Eigenvector Centrality](./eigenvector_centrality.md)|[Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Eigenvector](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Eigenvector.ipynb)|Measure of connectivity to other important vertices (which also have high connectivity) often referred to as the influence measure of a vertex| +|[Katz Centrality](./katz_centrality.md)|[Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Katz](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Katz.ipynb) |Similar to Eigenvector but has tweaks to measure more weakly connected graph | +|Pagerank|[Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Pagerank](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/Pagerank.ipynb) |Classified as both a link analysis and centrality measure by quantifying incoming links from central vertices. | + +[System Requirements](https://github.com/rapidsai/cugraph/blob/main/notebooks/README.md#requirements) + -[System Requirements](../../README.md#requirements) | Author Credit | Date | Update | cuGraph Version | Test Hardware | | --------------|------------|------------------|-----------------|----------------| diff --git a/docs/cugraph/source/graph_support/algorithms/Similarity.md b/docs/cugraph/source/graph_support/algorithms/Similarity.md index 450beb373a2..18c0a94d519 100644 --- a/docs/cugraph/source/graph_support/algorithms/Similarity.md +++ b/docs/cugraph/source/graph_support/algorithms/Similarity.md @@ -15,9 +15,9 @@ Manipulation of the data before or after the graph analytic is not covered here. |Algorithm |Notebooks Containing |Description | | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -|[Jaccard Smiliarity](./jaccard_similarity.md)| [Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) || -|[Overlap Similarity](./overlap_similarity.md)| [Overlap Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Overlap-Similarity.ipynb) || -|[Sorensen](./sorensen_coefficient.md)|[Sorensen Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Sorensen_coefficient.ipynb)|| +|[Jaccard Smiliarity](./jaccard_similarity.html)| [Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) || +|[Overlap Similarity](./overlap_similarity.html)| [Overlap Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Overlap-Similarity.ipynb) || +|[Sorensen](./sorensen_coefficient.html)|[Sorensen Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Sorensen_coefficient.ipynb)|| |Personal Pagerank|[Pagerank](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/Pagerank.ipynb) || diff --git a/docs/cugraph/source/images/cugraph_logo_2.png b/docs/cugraph/source/images/cugraph_logo_2.png new file mode 100644 index 00000000000..62dd79c4b98 Binary files /dev/null and b/docs/cugraph/source/images/cugraph_logo_2.png differ diff --git a/docs/cugraph/source/index.rst b/docs/cugraph/source/index.rst index 4690669203a..955eb6d54db 100644 --- a/docs/cugraph/source/index.rst +++ b/docs/cugraph/source/index.rst @@ -1,19 +1,45 @@ -Welcome to RAPIDS Graph documentation -===================================== -RAPIDS Graph covers a range of graph libraries and packages, that includes: - -* cugraph: GPU-accelerated graph algorithms -* cugraph-ops: GPU-accelerated GNN aggregators and operators -* cugraph-service: multi-user, remote GPU-accelerated graph algorithm service -* cugraph-pyg: GPU-accelerated extensions for use with the PyG framework -* cugraph-dgl: GPU-accelerated extensions for use with the DGL framework -* wholegraph: shared memory-based GPU-accelerated GNN training - -cuGraph is a library of graph algorithms that seamlessly integrates into the RAPIDS data science ecosystem and allows the data scientist to easily call graph algorithms using data stored in GPU DataFrames, NetworkX Graphs, or even CuPy or SciPy sparse Matrices. +RAPIDS Graph documentation +========================== +.. image:: images/cugraph_logo_2.png + :width: 600 + +*Making graph analytics fast and easy regardless of scale* + + +.. list-table:: RAPIDS Graph covers a range of graph libraries and packages, that includes: + :widths: 25 25 25 + :header-rows: 1 + + * - Core + - GNN + - Extension + * - :abbr:`cugraph (Python wrapper with lots of convenience functions)` + - :abbr:`cugraph-ops (GNN aggregators and operators)` + - :abbr:`cugraph-service (Graph-as-a-service provides both Client and Server packages)` + * - :abbr:`pylibcugraph (light-weight Python wrapper with no guard rails)` + - :abbr:`cugraph-dgl (Accelerated extensions for use with the DGL framework)` + - + * - :abbr:`libcugraph (C++ API)` + - :abbr:`cugraph-pyg (Accelerated extensions for use with the PyG framework)` + - + * - :abbr:`libcugraph_etl (C++ renumbering function for strings)` + - :abbr:`wholegraph (Shared memory-based GPU-accelerated GNN training)` + - +.. +| + +~~~~~~~~~~~~ +Introduction +~~~~~~~~~~~~ +cuGraph is a library of graph algorithms that seamlessly integrates into the +RAPIDS data science ecosystem and allows the data scientist to easily call +graph algorithms using data stored in GPU DataFrames, NetworkX Graphs, or +even CuPy or SciPy sparse Matrices. Note: We are redoing all of our documents, please be patient as we update the docs and links +| .. toctree:: :maxdepth: 2 @@ -23,9 +49,8 @@ the docs and links installation/index tutorials/index graph_support/index + wholegraph/index references/index - dev_resources/index - releases/index api_docs/index Indices and tables diff --git a/docs/cugraph/source/installation/getting_cugraph.md b/docs/cugraph/source/installation/getting_cugraph.md index 509508c5283..625f2c64c27 100644 --- a/docs/cugraph/source/installation/getting_cugraph.md +++ b/docs/cugraph/source/installation/getting_cugraph.md @@ -9,7 +9,7 @@ There are 4 ways to get cuGraph packages: 1. [Quick start with Docker Repo](#docker) 2. [Conda Installation](#conda) 3. [Pip Installation](#pip) -4. [Build from Source](#SOURCE) +4. [Build from Source](./source_build.md)
diff --git a/docs/cugraph/source/wholegraph/basics/index.rst b/docs/cugraph/source/wholegraph/basics/index.rst new file mode 100644 index 00000000000..429fe35d601 --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/index.rst @@ -0,0 +1,11 @@ +====== +Basics +====== + + +.. toctree:: + :maxdepth: 2 + + wholegraph_intro + wholememory_intro + wholememory_implementation_details diff --git a/docs/cugraph/source/wholegraph/basics/wholegraph_intro.md b/docs/cugraph/source/wholegraph/basics/wholegraph_intro.md new file mode 100644 index 00000000000..360f8e0e36b --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/wholegraph_intro.md @@ -0,0 +1,135 @@ +# WholeGraph Introduction +WholeGraph helps train large-scale Graph Neural Networks(GNN). +WholeGraph provides underlying storage structure called WholeMemory. +WholeMemory is a Tensor like storage and provides multi-GPU support. +It is optimized for NVLink systems like DGX A100 servers. +By working together with cuGraph, cuGraph-Ops, cuGraph-DGL, cuGraph-PyG, and upstream DGL and PyG, +it will be easy to build GNN applications. + +## WholeMemory +WholeMemory can be regarded as a whole view of GPU memory. +WholeMemory exposes a handle of the memory instance no matter how the underlying data is stored across multiple GPUs. +WholeMemory assumes that separate process is used to control each GPU. + +### WholeMemory Basics +To define WholeMemory, we need to specify the following: + +#### 1. Specify the set of GPU to handle the Memory + +Since WholeMemory is owned by a set of GPUs, you must specify the set of GPUs. +This is done by creating [WholeMemory Communicator](#wholememory-communicator) and specifying the WholeMemory Communicator when creating WholeMemory. + +#### 2. Specify the location of the memory + +Although WholeMemory is owned by a set of GPUs, the memory itself can be located in host memory or in device memory. +The location of the memory need to be specified, two types of locations can be specified. + +- **Host memory**: will use pinned host memory as underlying storage. +- **Device memory**: will use GPU device memory as underlying storage. + +#### 3. Specify the address mapping mode of the memory + +As WholeMemory is owned by multiple GPUs, each GPU will access the whole memory space, so we need address mapping. +There are three types of address mapping modes (also known as WholeMemory types), they are: + +- **Continuous**: All memory from each GPU will be mapped into a single continuous memory address space for each GPU. + In this mode, each GPU can directly access the whole memory using a single pointer and offset, just like using normal + device memory. Software will see no difference. Hardware peer to peer access will handle the underlying communication. + +- **Chunked**: Memory from each GPU will be mapped into different memory chunks, one chunk for each GPU. + In this mode, direct access is also supported, but not using a single pointer. Software will see the chunked memory. + However, an abstract layer may help to hide this. + +- **Distributed**: Memory from other GPUs are not mapped into current GPU, so no direct access is supported. + To access memory of other GPU, explicit communication is needed. + +To learn more details about WholeMemory locations and WholeMemory types, please refer to +[WholeMemory Implementation Details](wholememory_implementation_details.md) + +### WholeMemory Communicator +WholeMemory Communicator has two main purpose: + +- **Defines a set of GPUs which works together on WholeMemory.** WholeMemory Communicator is created by all GPUs that + wants to work together. A WholeMemory Communicator can be reused as long as the GPU set needed is the same. +- **Provides underlying communication channel needed by WholeMemory.** WholeMemory may need commuincator between GPUs + during the WholeMemory creation and some OPs on some types of WholeMemory. + +To Create WholeMemory Communicator, a WholeMemory Unique ID needs to be created first, it is usually created by the first GPU in the set of GPUs, and then broadcasted to all GPUs that want to work together. Then all GPUs in this communicator +will call WholeMemory Communicator creation function using this WholeMemory Unique ID, and the rank of current GPU as +well as all GPU count. + +### WholeMemory Granularity +As underlying storage may be partitioned into multiple GPUs physically, this is usually not wanted inside one single +user data block. To help on this, when creating WholeMemory, the granularity of data can be specified. Then the +WholeMemory is considered as multiple block of the same granularity and will not get split inside the granularity. + +### WholeMemory Mapping +As WholeMemory provides a whole view of memory to GPU, to access WholeMemory, mapping is usually needed. +Different types of WholeMemory have different mapping methods supported as their names. +Some mappings supported include +- All the WholeMemory types support mapping the memory range that local GPU is responsible for. + That is, each rank can directly access "Local" memory in all types of WholeMemory. + Here "Local" memory doesn't have to be on current GPU's memory, it can be on host memory or even maybe on other GPU, + but it is guaranteed to be directly accessed by current GPU. +- Chunked and Continuous WholeMemory also support Chunked mapping. That is, memory of all GPUs can be mapped into + current GPU, one continuous chunk for one GPU. Each chunk can be directly accessed by current GPU. But the memory of + different chunks are not guaranteed to be continuous. +- Continuous WholeMemory can be mapped into continuous memory space. That is, memory of all GPUs are mapped into a + single range of virtual memory, accessing to different position of this memory will physically access to different + GPUs. This mapping will be handled by hardware (CPU pagetable or GPU pagetable). + +### Operations on WholeMemory +There are some operations that can be performed on WholeMemory. They are based on the mapping of WholeMemory. +#### Local Operation +As all WholeMemory supports mapping of local memory, so operation on local memory is supported. The operation can be +either read or write. Just use it as GPU memory of current device is OK. +#### Load and Store +To facilitate file operation, Load / Store WholeMemory from file or to file is supported. WholeMemory uses raw binary +file format for disk operation. For Load, the input file can be a single file or a list of files, if it is a list, they +will be logically concatenated together and then loaded. For store, each GPU stores its local memory to file, producing +a list of files. +#### Gather and Scatter +WholeMemory also supports Gather / Scatter operation, usually they operate on a +[WholeMemory Tensor](#wholememory-tensor). + +### WholeMemory Tensor +Compared to PyTorch, WholeMemory is like PyTorch Storage while a WholeMemory Tensor is like a PyTorch Tensor. +For now, WholeMemory supports only 1D and 2D tensors, or arrays and matrices. Only first dimension is partitioned. + +### WholeMemory Embedding +WholeMemory Embedding is just like a 2D WholeMemory Tensor, with two features added. They support cache and sparse +optimizers. +#### Cache Support +To create WholeMemory Embedding with a cache, WholeMemory CachePolicy needs to be be created first. WholeMemoryCachePolicy can be created with following fields: +- **WholeMemory Communicator**: WholeMemory CachePolicy also needs WholeMemory Communicator. + WholeMemory Communicator defines the set of GPUs that cache all the Embedding. + It can be the same as the WholeMemory Communicator used to create WholeMemory Embedding. +- **WholeMemory type**: WholeMemory CachePolicy uses WholeMemory type to specify the WholeMemory type of cache. +- **WholeMemory location**: WholeMemory CachePolicy use WholeMemory location to specify the location of the cache. +- **Access type**: Access type can be readonly or readwrite. +- **Cache ratio**: Specify how much memory the cache will use. This ratio is computed for each GPU set that caches the + whole embedding. + +The two most commonly used caches are: +- **Device cached host memory**: When the WholeMemory Communicator for Cache Policy is the same as the WholeMemory + Communicator used to create WholeMemory Embedding, it means that the cache has same GPU set as WholeMemory Embedding. + So each GPU just caches its own part of raw Embedding. + Most commonly, when raw WholeMemory Embedding is located on host memory, and the cache is on device + memory, each GPU just caches its own part of host memory. +- **Local cached global memory**: The WholeMemory Communicator of WholeMemory CachePolicy can also be a subset of the + WholeMemory Communicator of WholeMemory Embedding. In this case, the subset of GPUs together cache all the embeddings. + Normally, when raw WholeMemory Embedding is partitioned on different machine nodes, and we + want to cache some embeddings in local machine or local GPU, then the subset of GPU can be all the GPUs in the local + machine. For local cached global memory, only readonly is supported. + +#### WholeMemory Embedding Sparse Optimizer +Another feature of WholeMemory Embedding is that WholeMemory Embedding supports embedding training. +To efficiently train large embedding tables, a sparse optimizer is needed. +WholeMemory Embedding Sparse Optimizer can run on a cached or noncached WholeMemory Embedding. +Currently supported optimizers include SGD, Adam, RMSProp and AdaGrad. + +## Graph Structure +Graph structure in WholeGraph is also based on WholeMemory. +In WholeGraph, graph is stored in [CSR format](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format)). +Both ROW_INDEX (noted as `csr_row_ptr`) and COL_INDEX (notated as `csr_col_ind`) are stored in a +WholeMemory Tensor. So loading Graph Structure can use [WholeMemory Tensor Loading mechanism](#load-and-store). diff --git a/docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md b/docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md new file mode 100644 index 00000000000..a5541109c4f --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md @@ -0,0 +1,58 @@ +# WholeMemory Implementation Details +As described in [WholeMemory Introduction](wholegraph_intro.md), there are two WholeMemory location and three +WholeMemory types. So there will be total six WholeMemory. + +| Type | CONTINUOUS | CONTINUOUS | CHUNKED | CHUNKED | DISTRIBUTED | DISTRIBUTED | +|:-------------:|:-----------:|:----------:|:---------:|:---------:|:-----------:|:-----------:| +| Location | DEVICE | HOST | DEVICE | HOST | DEVICE | HOST | +| Allocated by | EACH | FIRST | EACH | FIRST | EACH | EACH | +| Allocate API | Driver | Host | Runtime | Host | Runtime | Runtime | +| IPC Mapping | Unix fd | mmap | cudaIpc | mmap | No IPC map | No IPC map | + +For "Continuous" and "Chunked" types of WholeMemory, all memory is mapped to each GPU, +so these two types are all "Mapped" WholeMemory, in contrast to "Distributed" WholeMemory where all are not mapped. + +## WholeMemory Layout +Since the underlying memory of a single WholeMemory object may be on multiple GPU devices, the WholeGraph library will +partition data into these GPU devices. +The partition method guarantees that each GPU can access one continuous part of the entire memory. +Here "can access" means can directly access from CUDA kernels, but the memory doesn't have to be physically on that GPU. +For example,it can be on host memory or other GPU's device memory that can be access using P2P. +In that case the stored data has its own granularity that shouldn't be split. Data granularity can be specified while +creating WholeMemory. Then each data granularity can be considered as a block of data. + +The follow figure shows the layout of 15 data block over 4 GPUs. +![WholeMemory Layout](../imgs/general_wholememory.png) + +For WholeMemory Tensors, they can be 1D or 2D tensors. +For 1D tensor, data granularity is one element. For 2D tensor, data granularity is its 1D tensor. +The layout will be like this: +![WholeMemory Tensor Layout](../imgs/wholememory_tensor.png) + +## WholeMemory Allocation +As there are six types of WholeMemory, the allocation process of each type are as follows: + +### Device Continuous WholeMemory +For Device Continuous WholeMemory, first a range of virtual address space is reserved in each GPU, which covers the +entire memory range. Then a part of pyhsical memory is allocated in each GPU, as shown in the following figure. +![Device Continuous WholeMemory Allocation Step 1](../imgs/device_continuous_wholememory_step1.png) +After that, each GPU gathers all the memory handles from all GPUs, and maps them to the reserved address space. +![Device Continuous WholeMemory Allocation Step 2](../imgs/device_continuous_wholememory_step2.png) + +### Device Chunked WholeMemory +For Device Chunked WholeMemory, first each GPU allocates its own part of memory using CUDA runtime API, this will create +both a virtual address space and physical memory for its own memory. +![Device Chunked WholeMemory Allocation Step 1](../imgs/device_chunked_wholememory_step1.png) +Each GPU gathers the Ipc handle of memory from all other GPUs, and maps that into its own virtual address space. +![Device Chunked WholeMemory Allocation Step 2](../imgs/device_chunked_wholememory_step2.png) + +### Host Mapped WholeMemory +For Host, Continuous and Chunked are using the same method. First, rank and allocate the host physical and share that to all +ranks. +![Host Mapped WholeMemory Allocation Step 1](../imgs/host_mapped_wholememory_step1.png) +Then each rank registers that host memory to GPU address space. +![Host Mapped WholeMemory Allocation Step 2](../imgs/host_mapped_wholememory_step2.png) + +### Distributed WholeMemory +For Distributed WholeMemory, each GPU just malloc its own part of memory, no need to share to other GPUs. +![Distributed WholeMemory Allocation](../imgs/distributed_wholememory.png) diff --git a/docs/cugraph/source/wholegraph/basics/wholememory_intro.md b/docs/cugraph/source/wholegraph/basics/wholememory_intro.md new file mode 100644 index 00000000000..7209da9471c --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/wholememory_intro.md @@ -0,0 +1,123 @@ +## WholeMemory +WholeMemory can be regarded as a whole view of GPU memory. +WholeMemory exposes a handle to the memory instance no matter how the underlying data is stored across multiple GPUs. +WholeMemory assumes that a separate process is used to control each GPU. + +### WholeMemory Basics +To define WholeMemory, we need to specify the following: + +#### 1. Specify the set of GPU to handle the Memory + +As WholeMemory is owned by a set of GPUs, so the set of GPUs need to be specified. +This is done by creating [WholeMemory Communicator](#wholememory-communicator) and specify the WholeMemory Communicator +when creating WholeMemory. + +#### 2. Specify the location of the memory + +Although WholeMemory is owned by a set of GPUs, the memory itself can be located on host memory or on device memory. +So the location of the memory needs to be specified. Two types of location can be specified. + +- **Host memory**: will use pinned host memory as underlying storage. +- **Device memory**: will use GPU device memory as underlying storage. + +#### 3. Specify the address mapping mode of the memory + +As WholeMemory is owned by multiple GPUs, each GPU will access the whole memory space, so we need address mapping. +There are three types of address mapping modes (also known as WholeMemory types), they are: + +- **Continuous**: All memory from each GPU will be mapped into a single continuous memory address space for each GPU. + In this mode, each GPU can directly access the whole memory using a single pointer and offset, just like using normal + device memory. Software will see no difference. Hardware peer-to-peer access will handle the underlying communication. + +- **Chunked**: Memory from each GPU will be mapped into different memory chunks, one chunk for each GPU. + In this mode, direct access is also supported, but not using a single pointer. Software will see the chunked memory. + However, an abstract layer can hide this. + +- **Distributed**: Memory from other GPUs is not mapped into current GPU, so no direct access is supported. + To access memory of another GPU, explicit communication is needed. + +If you would like to know more details about WholeMemory locations and WholeMemory types, please refer to +[WholeMemory Implementation Details](wholememory_implementation_details.md) + +### WholeMemory Communicator +WholeMemory Communicator has two main purpose: + +- **Defines a set of GPUs which works together on WholeMemory.** WholeMemory Communicator is created by all GPUs that + wants to work together. A WholeMemory Communicator can be reused as long as the GPU set needed is the same. +- **Provides underlying communication channel needed by WholeMemory.** WholeMemory may need commuincator between GPUs + during the WholeMemory creation and some OPs on some types of WholeMemory. + +To Create WholeMemory Communicator, a WholeMemory Unique ID need to be created first, it is usually created by the first +GPU in the set of GPUs, and then broadcasted to all GPUs that want to work together. Then all GPUs in this communicator +will call WholeMemory Communicator creation function using this WholeMemory Unique ID, and the rank of current GPU as +well as all GPU count. + +### WholeMemory Granularity +As underlying storage may be physically partitioned into multiple GPUs, it is usually not wanted inside one single +user data block. To help with this, when creating WholeMemory, the granularity of data can be specified. Therefore +WholeMemory is considered as multiple blocks of the same granularity and will not get split inside the granularity. + +### WholeMemory Mapping +Since WholeMemory provides a whole view of memory to GPU, mapping is usually needed to access WholeMemory. +Different types of WholeMemory have different mapping methods supported as their names. +Some mappings supported include: +- All the WholeMemory types support mapping the memory range that local GPU is responsible for. + That is, each rank can directly access "Local" memory in all types of WholeMemory. + Here "Local" memory doesn't have to be on current GPU's memory, it can be on host memory or even maybe on other GPU, + but it is guaranteed to be directly accessed by current GPU. +- Chunked and Continuous WholeMemory also support Chunked mapping. That is, memory of all GPUs can be mapped into + current GPU, one continuous chunk for one GPU. Each chunk can be directly accessed by current GPU. But the memory of + different chunks are not guaranteed to be continuous. +- Continuous WholeMemory can be mapped into continuous memory space. That is, memory of all GPUs are mapped into a + single range of virtual memory, accessing different positions of this memory will physically access different + GPUs. This mapping will be handled by hardware (CPU pagetable or GPU pagetable). + +### Operations on WholeMemory +There are some operations that can be performed on WholeMemory. They are based on the mapping of WholeMemory. +#### Local Operation +As all WholeMemory supports mapping of local memory, so operation on local memory is supported. The operation can be +either read or write. Just use it as GPU memory of current device is OK. +#### Load / Store +To facilitate file operation, Load / Store WholeMemory from file or to file is supported. WholeMemory use raw binary +file format for disk operation. For Load, the input file can be single file or a list of files, if it is a list, they +will be logically concatenated together and then loaded. For store, each GPU stores its local memory to file, producing +a list of files. +#### Gather / Scatter +WholeMemory also supports Gather / Scatter operations, usually they operate on a +[WholeMemory Tensor](#wholememory-tensor). + +### WholeMemory Tensor +Compared to PyTorch, WholeMemory is like PyTorch Storage while WholeMemory Tensor is like PyTorch Tensor. +For now, WholeMemory supports only 1D and 2D tensor, or array and matrix. Only first dimension is partitioned. + +### WholeMemory Embedding +WholeMemory Embedding is just like 2D WholeMemory Tensor, with cache support and sparse optimizer support added. +#### Cache Support +WholeMemory Embedding supports cache. To create WholeMemory Embedding with cache, WholeMemory CachePolicy need first be +created. WholeMemoryCachePolicy can be created with following fields: +- **WholeMemory Communicator**: WholeMemory CachePolicy also need WholeMemory Communicator. + This WholeMemory Communicator defines the set of GPUs that cache the all the Embedding. + It can be the same as the WholeMemory Communicator used to create WholeMemory Embedding. +- **WholeMemory type**: WholeMemory CachePolicy uses WholeMemory type to specify the WholeMemory type of the cache. +- **WholeMemory location**: WholeMemory CachePolicy uses WholeMemory location to specify the location of the cache. +- **Access type**: Access type can be readonly or readwrite. +- **Cache ratio**: Specify how much memory the cache will use. This ratio is computed for each GPU set that caches the + whole embedding. + +There are two most commonly used caches. They are: +- **Device cached host memory**: When the WholeMemory Communicator for Cache Policy is the same as the WholeMemory + Communicator used to create WholeMemory Embedding, it means that cache has the same GPU set as WholeMemory Embedding. + So each GPU just cache its own part of raw Embedding. + Normally, when raw WholeMemory Embedding is located on host memory, and the cache is on device + memory, each GPU just caches its own part of host memory. +- **Local cached global memory**: The WholeMemory Communicator of WholeMemory CachePolicy can also be a subset of the + WholeMemory Communicator of WholeMemory Embedding. In this case, the subset of GPUs together cache all the embeddings. + Typically, raw WholeMemory Embedding is partitioned on different machine nodes, and we + want to cache some embeddings in local machine or local GPU, then the subset of GPUs can be all the GPUs on the local + machine. For local cached global memory supports just readonly. + +#### WholeMemory Embedding Sparse Optimizer +Another feature of WholeMemory Embedding is that WholeMemory Embedding supports embedding training. +To efficiently train large embedding tables, a sparse optimizer is needed. +The WholeMemory Embedding Sparse Optimizer can run on cached or non-cached WholeMemory Embedding. +Currently supported optimizers include SGD, Adam, RMSProp and AdaGrad. diff --git a/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step1.png b/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step1.png new file mode 100644 index 00000000000..b8a0447e6fb Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step1.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step2.png b/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step2.png new file mode 100644 index 00000000000..8b203ce2246 Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step2.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step1.png b/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step1.png new file mode 100644 index 00000000000..46ecd1f14e7 Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step1.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step2.png b/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step2.png new file mode 100644 index 00000000000..b773b1ef6e9 Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step2.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/distributed_wholememory.png b/docs/cugraph/source/wholegraph/imgs/distributed_wholememory.png new file mode 100644 index 00000000000..e6bbe9f13e9 Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/distributed_wholememory.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/general_wholememory.png b/docs/cugraph/source/wholegraph/imgs/general_wholememory.png new file mode 100644 index 00000000000..3ece02b007b Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/general_wholememory.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step1.png b/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step1.png new file mode 100644 index 00000000000..aad8caf0d07 Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step1.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step2.png b/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step2.png new file mode 100644 index 00000000000..20597f3e515 Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step2.png differ diff --git a/docs/cugraph/source/wholegraph/imgs/wholememory_tensor.png b/docs/cugraph/source/wholegraph/imgs/wholememory_tensor.png new file mode 100644 index 00000000000..e725d6c28ed Binary files /dev/null and b/docs/cugraph/source/wholegraph/imgs/wholememory_tensor.png differ diff --git a/docs/cugraph/source/wholegraph/index.rst b/docs/cugraph/source/wholegraph/index.rst new file mode 100644 index 00000000000..2a69544b4c9 --- /dev/null +++ b/docs/cugraph/source/wholegraph/index.rst @@ -0,0 +1,14 @@ +WholeGraph +========== +RAPIDS WholeGraph has following package: + +* pylibwholegraph: shared memory-based GPU-accelerated GNN training + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + basics/index + installation/index + diff --git a/docs/cugraph/source/wholegraph/installation/container.md b/docs/cugraph/source/wholegraph/installation/container.md new file mode 100644 index 00000000000..3a2c627c56a --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/container.md @@ -0,0 +1,29 @@ +# Build Container for WholeGraph +To run WholeGraph or build WholeGraph from source, set up the environment first. +We recommend using Docker images. +For example, to build the WholeGraph base image from the NGC pytorch 22.10 image, you can follow `Dockerfile`: +```dockerfile +FROM nvcr.io/nvidia/pytorch:22.10-py3 + +RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y lsb-core software-properties-common wget libspdlog-dev + +#RUN remove old cmake to update +RUN conda remove --force -y cmake +RUN rm -rf /usr/local/bin/cmake && rm -rf /usr/local/lib/cmake && rm -rf /usr/lib/cmake + +RUN apt-key adv --fetch-keys https://apt.kitware.com/keys/kitware-archive-latest.asc && \ + export LSB_CODENAME=$(lsb_release -cs) && \ + apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ ${LSB_CODENAME} main" && \ + apt update && apt install -y cmake + +# update py for pytest +RUN pip3 install -U py +RUN pip3 install Cython setuputils3 scikit-build nanobind pytest-forked pytest +``` + +To run GNN applications, you may also need cuGraphOps, DGL and/or PyG libraries to run the GNN layers. +You may refer to [DGL](https://www.dgl.ai/pages/start.html) or [PyG](https://pytorch-geometric.readthedocs.io/en/latest/notes/installation.html) +For example, to install DGL, you may need to add: +```dockerfile +RUN pip3 install dgl -f https://data.dgl.ai/wheels/cu118/repo.html +``` diff --git a/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md new file mode 100644 index 00000000000..5b2072b0523 --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md @@ -0,0 +1,48 @@ + +# Getting the WholeGraph Packages + +Start by reading the [RAPIDS Instalation guide](https://docs.rapids.ai/install) +and checkout the [RAPIDS install selector](https://rapids.ai/start.html) for a pick list of install options. + + +There are 4 ways to get WholeGraph packages: +1. [Quick start with Docker Repo](#docker) +2. [Conda Installation](#conda) +3. [Pip Installation](#pip) +4. [Build from Source](./source_build.md) + + +
+ +## Docker +The RAPIDS Docker containers (as of Release 23.10) contain all RAPIDS packages, including WholeGraph, as well as all required supporting packages. To download a container, please see the [Docker Repository](https://hub.docker.com/r/rapidsai/rapidsai/), choosing a tag based on the NVIDIA CUDA version you’re running. This provides a ready to run Docker container with example notebooks and data, showcasing how you can utilize all of the RAPIDS libraries. + +
+ + +## Conda +It is easy to install WholeGraph using conda. You can get a minimal conda installation with [Miniconda](https://conda.io/miniconda.html) or get the full installation with [Anaconda](https://www.anaconda.com/download). + +WholeGraph conda packages + * libwholegraph + * pylibwholegraph + +Replace the package name in the example below to the one you want to install. + + +Install and update WholeGraph using the conda command: + +```bash +conda install -c rapidsai -c conda-forge -c nvidia wholegraph cudatoolkit=11.8 +``` + +
+ +## PIP +wholegraph, and all of RAPIDS, is available via pip. + +``` +pip install wholegraph-cu11 --extra-index-url=https://pypi.nvidia.com +``` + +
diff --git a/docs/cugraph/source/wholegraph/installation/index.rst b/docs/cugraph/source/wholegraph/installation/index.rst new file mode 100644 index 00000000000..09f1cb44a24 --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/index.rst @@ -0,0 +1,9 @@ +Installation +============ + +.. toctree:: + :maxdepth: 2 + + getting_wholegraph + container + source_build diff --git a/docs/cugraph/source/wholegraph/installation/source_build.md b/docs/cugraph/source/wholegraph/installation/source_build.md new file mode 100644 index 00000000000..c468048c351 --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/source_build.md @@ -0,0 +1,187 @@ +# Building from Source + +The following instructions are for users wishing to build wholegraph from source code. These instructions are tested on supported distributions of Linux,CUDA, +and Python - See [RAPIDS Getting Started](https://rapids.ai/start.html) for a list of supported environments. +Other operating systems _might be_ compatible, but are not currently tested. + +The wholegraph package includes both a C/C++ CUDA portion and a python portion. Both libraries need to be installed in order for cuGraph to operate correctly. +The C/C++ CUDA library is `libwholegraph` and the python library is `pylibwholegraph`. + +## Prerequisites + +__Compiler__: +* `gcc` version 11.0+ +* `nvcc` version 11.0+ +* `cmake` version 3.26.4+ + +__CUDA__: +* CUDA 11.8+ +* NVIDIA driver 450.80.02+ +* Pascal architecture or better + +You can obtain CUDA from [https://developer.nvidia.com/cuda-downloads](https://developer.nvidia.com/cuda-downloads). + +__Other Packages__: +* ninja +* nccl +* cython +* setuputils3 +* scikit-learn +* scikit-build +* nanobind>=0.2.0 + +## Building wholegraph +To install wholegraph from source, ensure the dependencies are met. + +### Clone Repo and Configure Conda Environment +__GIT clone a version of the repository__ + + ```bash + # Set the location to wholegraph in an environment variable WHOLEGRAPH_HOME + export WHOLEGRAPH_HOME=$(pwd)/wholegraph + + # Download the wholegraph repo - if you have a forked version, use that path here instead + git clone https://github.com/rapidsai/wholegraph.git $WHOLEGRAPH_HOME + + cd $WHOLEGRAPH_HOME + ``` + +__Create the conda development environment__ + +```bash +# create the conda environment (assuming in base `wholegraph` directory) + +# for CUDA 11.x +conda env create --name wholegraph_dev --file conda/environments/all_cuda-118_arch-x86_64.yaml + +# activate the environment +conda activate wholegraph_dev + +# to deactivate an environment +conda deactivate +``` + + - The environment can be updated as development includes/changes the dependencies. To do so, run: + + +```bash + +# Where XXX is the CUDA version +conda env update --name wholegraph_dev --file conda/environments/all_cuda-XXX_arch-x86_64.yaml + +conda activate wholegraph_dev +``` + + +### Build and Install Using the `build.sh` Script +Using the `build.sh` script make compiling and installing wholegraph a +breeze. To build and install, simply do: + +```bash +$ cd $WHOLEGRAPH_HOME +$ ./build.sh clean +$ ./build.sh libwholegraph +$ ./build.sh pylibwholegraph +``` + +There are several other options available on the build script for advanced users. +`build.sh` options: +```bash +build.sh [ ...] [ ...] + where is: + clean - remove all existing build artifacts and configuration (start over). + uninstall - uninstall libwholegraph and pylibwholegraph from a prior build/install (see also -n) + libwholegraph - build the libwholegraph C++ library. + pylibwholegraph - build the pylibwholegraph Python package. + tests - build the C++ (OPG) tests. + benchmarks - build benchmarks. + docs - build the docs + and is: + -v - verbose build mode + -g - build for debug + -n - no install step + --allgpuarch - build for all supported GPU architectures + --cmake-args=\\\"\\\" - add arbitrary CMake arguments to any cmake call + --compile-cmd - only output compile commands (invoke CMake without build) + --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) + -h | --h[elp] - print this text + + default action (no args) is to build and install 'libwholegraph' then 'pylibwholegraph' targets + +examples: +$ ./build.sh clean # remove prior build artifacts (start over) +$ ./build.sh + +# make parallelism options can also be defined: Example build jobs using 4 threads (make -j4) +$ PARALLEL_LEVEL=4 ./build.sh libwholegraph + +Note that the libraries will be installed to the location set in `$PREFIX` if set (i.e. `export PREFIX=/install/path`), otherwise to `$CONDA_PREFIX`. +``` + + +## Building each section independently +### Build and Install the C++/CUDA `libwholegraph` Library +CMake depends on the `nvcc` executable being on your path or defined in `$CUDACXX`. + +This project uses cmake for building the C/C++ library. To configure cmake, run: + + ```bash + # Set the location to wholegraph in an environment variable WHOLEGRAPH_HOME + export WHOLEGRAPH_HOME=$(pwd)/wholegraph + + cd $WHOLEGRAPH_HOME + cd cpp # enter cpp directory + mkdir build # create build directory + cd build # enter the build directory + cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX + + # now build the code + make -j # "-j" starts multiple threads + make install # install the libraries + ``` +The default installation locations are `$CMAKE_INSTALL_PREFIX/lib` and `$CMAKE_INSTALL_PREFIX/include/wholegraph` respectively. + +### Building and installing the Python package + +Build and Install the Python packages to your Python path: + +```bash +cd $WHOLEGRAPH_HOME +cd python +cd pylibwholegraph +python setup.py build_ext --inplace +python setup.py install # install pylibwholegraph +``` + +## Run tests + +Run either the C++ or the Python tests with datasets + + - **Python tests with datasets** + + ```bash + cd $WHOLEGRAPH_HOME + cd python + pytest + ``` + + - **C++ stand alone tests** + + From the build directory : + + ```bash + # Run the tests + cd $WHOLEGRAPH_HOME + cd cpp/build + gtests/PARALLEL_UTILS_TESTS # this is an executable file + ``` + + +Note: This conda installation only applies to Linux and Python versions 3.8/3.10. + +## Creating documentation + +Python API documentation can be generated from _./docs/wholegraph directory_. Or through using "./build.sh docs" + +## Attribution +Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index c32dc74da40..2c1dd855cb5 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -12,7 +12,7 @@ # the License. # ============================================================================= if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CUGRAPH_RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.10/RAPIDS.cmake + file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.12/RAPIDS.cmake ${CMAKE_CURRENT_BINARY_DIR}/CUGRAPH_RAPIDS.cmake ) endif() diff --git a/img/Stack2.png b/img/Stack2.png index 132e85c9d15..97f1979c2d8 100644 Binary files a/img/Stack2.png and b/img/Stack2.png differ diff --git a/notebooks/algorithms/centrality/Centrality.ipynb b/notebooks/algorithms/centrality/Centrality.ipynb index d19dd646b15..e470febb975 100644 --- a/notebooks/algorithms/centrality/Centrality.ipynb +++ b/notebooks/algorithms/centrality/Centrality.ipynb @@ -36,22 +36,34 @@ "__Degree Centrality__
\n", "Degree centrality is based on the notion that whoever has the most connections must be important. \n", "\n", - "$C_d(v) = \\frac{{\\text{{degree of vertex }} v}}{{\\text{{total number of vertices}} - 1}}$\n", - "\n", + "$C_{degree}(v) = \\frac{{\\text{degree of vertex} \\ v}}{{\\text{total number of vertices} - 1}}$\n", "\n", + "See:\n", + "* [Degree (graph theory) on Wikipedia](https://en.wikipedia.org/wiki/Degree_(graph_theory)) for more details on the algorithm.\n", + "* [Learn more about Degree Centrality](https://www.sci.unich.it/~francesc/teaching/network/degree.html)\n", "\n", - "___Closeness centrality – coming soon___
\n", + "__Closeness Centrality__
\n", "Closeness is a measure of the shortest path to every other node in the graph. A node that is close to every other node, can reach over other node in the fewest number of hops, means that it has greater influence on the network versus a node that is not close.\n", "\n", + "$C_{closeness}(v)=\\frac{n-1}{\\sum_{t} d(v,t)}$\n", + "\n", + "See:\n", + "* [Closeness Centrality on Wikipedia](https://en.wikipedia.org/wiki/Closeness_centrality) for more details on the algorithm.\n", + "* [Learn more about Closeness Centrality](https://www.sci.unich.it/~francesc/teaching/network/closeness.html)\n", + "\n", "__Betweenness Centrality__
\n", "Betweenness is a measure of the number of shortest paths that cross through a node, or over an edge. A node with high betweenness means that it had a greater influence on the flow of information. \n", "\n", "Betweenness centrality of a node 𝑣 is the sum of the fraction of all-pairs shortest paths that pass through 𝑣\n", "\n", - "$C_{betweenness}=\\sum_{s \\neq v \\neq t} \\frac{\\sigma_{st}(v)}{\\sigma_{st}}$\n", + "$C_{betweenness}(v)=\\sum_{s \\neq v \\neq t} \\frac{\\sigma_{st}(v)}{\\sigma_{st}}$\n", "\n", "To speedup runtime of betweenness centrailty, the metric can be computed on a limited number of nodes (randomly selected) and then used to estimate the other scores. For this example, the graphs are relatively small (under 5,000 nodes) so betweenness on every node will be computed.\n", "\n", + "See:\n", + "* [Betweenness Centrality on Wikipedia](https://en.wikipedia.org/wiki/Betweenness_centrality) for more details on the algorithm.\n", + "* [Learn more about Betweenness Centrality](https://www.sci.unich.it/~francesc/teaching/network/betweeness.html)\n", + "\n", "__Katz Centrality__
\n", "Katz is a variant of degree centrality and of eigenvector centrality. \n", "Katz centrality is a measure of the relative importance of a node within the graph based on measuring the influence across the total number of walks between vertex pairs.\n", @@ -60,7 +72,7 @@ "\n", "\n", "See:\n", - "* [Katz on Wikipedia](https://en.wikipedia.org/wiki/Katz_centrality) for more details on the algorithm.\n", + "* [Katz Centrality on Wikipedia](https://en.wikipedia.org/wiki/Katz_centrality) for more details on the algorithm.\n", "* [Learn more about Katz Centrality](https://www.sci.unich.it/~francesc/teaching/network/katz.html)\n", "\n", "__Eigenvector Centrality__
\n", diff --git a/notebooks/algorithms/centrality/Degree.ipynb b/notebooks/algorithms/centrality/Degree.ipynb index e7535420b65..5a5213a904f 100644 --- a/notebooks/algorithms/centrality/Degree.ipynb +++ b/notebooks/algorithms/centrality/Degree.ipynb @@ -27,7 +27,7 @@ "\n", "See [Degree Centrality on Wikipedia](https://en.wikipedia.org/wiki/Degree_centrality) for more details on the algorithm.\n", "\n", - "$C_d(v) = \\frac{{\\text{{degree of vertex }} v}}{{\\text{{number of vertices in graph}} - 1}}$" + "$C_d(v) = \\frac{{\\text{degree of vertex } \\ v}}{{\\text{number of vertices in graph} - 1}}$" ] }, { diff --git a/notebooks/algorithms/centrality/Katz.ipynb b/notebooks/algorithms/centrality/Katz.ipynb index c94a14bb14a..08ee42df788 100755 --- a/notebooks/algorithms/centrality/Katz.ipynb +++ b/notebooks/algorithms/centrality/Katz.ipynb @@ -333,13 +333,6 @@ "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n", "___" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/notebooks/demo/nx_cugraph_demo.ipynb b/notebooks/demo/nx_cugraph_demo.ipynb new file mode 100644 index 00000000000..6e50370ed80 --- /dev/null +++ b/notebooks/demo/nx_cugraph_demo.ipynb @@ -0,0 +1,672 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# `nx-cugraph`: a NetworkX backend that provides GPU acceleration with RAPIDS cuGraph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook will demonstrate the `nx-cugraph` NetworkX backend using the NetworkX betweenness_centrality algorithm.\n", + "\n", + "## Background\n", + "Networkx version 3.0 introduced a dispatching mechanism that allows users to configure NetworkX to dispatch various algorithms to third-party backends. Backends can provide different implementations of graph algorithms, allowing users to take advantage of capabilities not available in NetworkX. `nx-cugraph` is a NetworkX backend provided by the [RAPIDS](https://rapids.ai) cuGraph project that adds GPU acceleration to greatly improve performance.\n", + "\n", + "## System Requirements\n", + "Using `nx-cugraph` with this notebook requires the following: \n", + "- NVIDIA GPU, Pascal architecture or later\n", + "- CUDA 11.2, 11.4, 11.5, 11.8, or 12.0\n", + "- Python versions 3.9, 3.10, or 3.11\n", + "- NetworkX >= version 3.2\n", + " - _NetworkX 3.0 supports dispatching and is compatible with `nx-cugraph`, but this notebook will demonstrate features added in 3.2_\n", + " - At the time of this writing, NetworkX 3.2 is only available from source and can be installed by following the [development version install instructions](https://github.com/networkx/networkx/blob/main/INSTALL.rst#install-the-development-version).\n", + "- Pandas\n", + "\n", + "More details about system requirements can be found in the [RAPIDS System Requirements documentation](https://docs.rapids.ai/install#system-req)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Installation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Assuming NetworkX >= 3.2 has been installed using the [development version install instructions](https://github.com/networkx/networkx/blob/main/INSTALL.rst#install-the-development-version), `nx-cugraph` can be installed using either `conda` or `pip`. \n", + "\n", + "#### conda\n", + "```\n", + "conda install -c rapidsai-nightly -c conda-forge -c nvidia nx-cugraph\n", + "```\n", + "#### pip\n", + "```\n", + "python -m pip install nx-cugraph-cu11 --extra-index-url https://pypi.nvidia.com\n", + "```\n", + "#### _Notes:_\n", + " * nightly wheel builds will not be available until the 23.12 release, therefore the index URL for the stable release version is being used in the pip install command above.\n", + " * Additional information relevant to installing any RAPIDS package can be found [here](https://rapids.ai/#quick-start).\n", + " * If you installed any of the packages described here since running this notebook, you may need to restart the kernel to have them visible to this notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Notebook Helper Functions\n", + "\n", + "A few helper functions will be defined here that will be used in order to help keep this notebook easy to read." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "def reimport_networkx():\n", + " \"\"\"\n", + " Re-imports networkx for demonstrating different backend configuration\n", + " options applied at import-time. This is only needed for demonstration\n", + " purposes since other mechanisms are available for runtime configuration.\n", + " \"\"\"\n", + " # Using importlib.reload(networkx) has several caveats (described here:\n", + " # https://docs.python.org/3/library/imp.html?highlight=reload#imp.reload)\n", + " # which result in backend configuration not being re-applied correctly.\n", + " # Instead, manually remove all modules and re-import\n", + " nx_mods = [m for m in sys.modules.keys()\n", + " if (m.startswith(\"networkx\") or m.startswith(\"nx_cugraph\"))]\n", + " for m in nx_mods:\n", + " sys.modules.pop(m)\n", + " import networkx\n", + " return networkx\n", + "\n", + "\n", + "from pathlib import Path\n", + "import requests\n", + "import gzip\n", + "import pandas as pd\n", + "def create_cit_patents_graph(verbose=True):\n", + " \"\"\"\n", + " Downloads the cit-Patents dataset (if not previously downloaded), reads\n", + " it, and creates a nx.DiGraph from it and returns it.\n", + " cit-Patents is described here:\n", + " https://snap.stanford.edu/data/cit-Patents.html\n", + " \"\"\"\n", + " url = \"https://snap.stanford.edu/data/cit-Patents.txt.gz\"\n", + " gz_file_name = Path(url.split(\"/\")[-1])\n", + " csv_file_name = Path(gz_file_name.stem)\n", + " if csv_file_name.exists():\n", + " if verbose: print(f\"{csv_file_name} already exists, not downloading.\")\n", + " else:\n", + " if verbose: print(f\"downloading {url}...\", end=\"\", flush=True)\n", + " req = requests.get(url)\n", + " open(gz_file_name, \"wb\").write(req.content)\n", + " if verbose: print(\"done\")\n", + " if verbose: print(f\"unzipping {gz_file_name}...\", end=\"\", flush=True)\n", + " with gzip.open(gz_file_name, \"rb\") as gz_in:\n", + " with open(csv_file_name, \"wb\") as txt_out:\n", + " txt_out.write(gz_in.read())\n", + " if verbose: print(\"done\")\n", + "\n", + " if verbose: print(\"reading csv to dataframe...\", end=\"\", flush=True)\n", + " pandas_edgelist = pd.read_csv(\n", + " csv_file_name.name,\n", + " skiprows=4,\n", + " delimiter=\"\\t\",\n", + " names=[\"src\", \"dst\"],\n", + " dtype={\"src\":\"int32\", \"dst\":\"int32\"},\n", + " )\n", + " if verbose: print(\"done\")\n", + " if verbose: print(\"creating NX graph from dataframe...\", end=\"\", flush=True)\n", + " G = nx.from_pandas_edgelist(\n", + " pandas_edgelist, source=\"src\", target=\"dst\", create_using=nx.DiGraph\n", + " )\n", + " if verbose: print(\"done\")\n", + " return G" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Running `betweenness_centrality`\n", + "Let's start by running `betweenness_centrality` on the Karate Club graph using the default NetworkX implementation." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Zachary's Karate Club\n", + "\n", + "Zachary's Karate Club is a small dataset consisting of 34 nodes and 78 edges which represent the friendships between members of a karate club. This dataset is small enough to make comparing results between NetworkX and `nx-cugraph` easy." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import networkx as nx\n", + "karate_club_graph = nx.karate_club_graph()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Having NetworkX compute the `betweenness_centrality` values for each node on this graph is quick and easy." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.51 ms ± 1.02 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "%%timeit global karate_nx_bc_results\n", + "karate_nx_bc_results = nx.betweenness_centrality(karate_club_graph)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Automatic GPU acceleration\n", + "When `nx-cugraph` is installed, NetworkX will detect it on import and make it available as a backend for APIs supported by that backend. However, NetworkX does not assume the user always wants to use a particular backend, and instead looks at various configuration mechanisms in place for users to specify how NetworkX should use installed backends. Since NetworkX was not configured to use a backend for the above `betweenness_centrality` call, it used the default implementation provided by NetworkX.\n", + "\n", + "The first configuration mechanism to be demonstrated below is the `NETWORKX_AUTOMATIC_BACKENDS` environment variable. This environment variable directs NetworkX to use the backend specified everywhere it's supported and does not require the user to modify any of their existing NetworkX code.\n", + "\n", + "To use it, a user sets `NETWORKX_AUTOMATIC_BACKENDS` in their shell to the backend they'd like to use. If a user has more than one backend installed, the environment variable can also accept a comma-separated list of backends, ordered by priority in which NetworkX should use them, where the first backend that supports a particular API call will be used. For example:\n", + "```\n", + "bash> export NETWORKX_AUTOMATIC_BACKENDS=cugraph\n", + "bash> python my_nx_app.py # uses nx-cugraph wherever possible, then falls back to default implementation where it's not.\n", + "```\n", + "or in the case of multiple backends installed\n", + "```\n", + "bash> export NETWORKX_AUTOMATIC_BACKENDS=cugraph,graphblas\n", + "bash> python my_nx_app.py # uses nx-cugraph if possible, then nx-graphblas if possible, then default implementation.\n", + "```\n", + "\n", + "NetworkX looks at the environment variable and the installed backends at import time, and will not re-examine the environment after that. Because `networkx` was already imported in this notebook, the `reimport_nx()` utility will be called after the `os.environ` dictionary is updated to simulate an environment variable being set in the shell.\n", + "\n", + "**Please note, this is only needed for demonstration purposes to compare runs both with and without fully-automatic backend use enabled.**" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"NETWORKX_AUTOMATIC_BACKENDS\"] = \"cugraph\"\n", + "nx = reimport_networkx()\n", + "# reimporting nx requires reinstantiating Graphs since python considers\n", + "# types from the prior nx import != types from the reimported nx\n", + "karate_club_graph = nx.karate_club_graph()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Once the environment is updated, re-running the same `betweenness_centrality` call on the same graph requires no code changes." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "43.9 ms ± 222 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit global karate_cg_bc_results\n", + "karate_cg_bc_results = nx.betweenness_centrality(karate_club_graph)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "We may see that the same computation actually took *longer* using `nx-cugraph`. This is not too surprising given how small the graph is, since there's a small amount of overhead to copy data to and from the GPU which becomes more obvious on very small graphs. We'll see with a larger graph how this overhead becomes negligible." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Results Comparison\n", + "\n", + "Let's examine the results of each run to see how they compare. \n", + "The `betweenness_centrality` results are a dictionary mapping vertex IDs to betweenness_centrality scores. The score itself is usually not as important as the relative rank of each vertex ID (e.g. vertex A is ranked higher than vertex B in both sets of results)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NX: (0, 0.437635), CG: (0, 0.437635)\n", + "NX: (33, 0.304075), CG: (33, 0.304075)\n", + "NX: (32, 0.145247), CG: (32, 0.145247)\n", + "NX: (2, 0.143657), CG: (2, 0.143657)\n", + "NX: (31, 0.138276), CG: (31, 0.138276)\n", + "NX: (8, 0.055927), CG: (8, 0.055927)\n", + "NX: (1, 0.053937), CG: (1, 0.053937)\n", + "NX: (13, 0.045863), CG: (13, 0.045863)\n", + "NX: (19, 0.032475), CG: (19, 0.032475)\n", + "NX: (5, 0.029987), CG: (5, 0.029987)\n", + "NX: (6, 0.029987), CG: (6, 0.029987)\n", + "NX: (27, 0.022333), CG: (27, 0.022333)\n", + "NX: (23, 0.017614), CG: (23, 0.017614)\n", + "NX: (30, 0.014412), CG: (30, 0.014412)\n", + "NX: (3, 0.011909), CG: (3, 0.011909)\n", + "NX: (25, 0.003840), CG: (25, 0.003840)\n", + "NX: (29, 0.002922), CG: (29, 0.002922)\n", + "NX: (24, 0.002210), CG: (24, 0.002210)\n", + "NX: (28, 0.001795), CG: (28, 0.001795)\n", + "NX: (9, 0.000848), CG: (9, 0.000848)\n", + "NX: (4, 0.000631), CG: (4, 0.000631)\n", + "NX: (10, 0.000631), CG: (10, 0.000631)\n", + "NX: (7, 0.000000), CG: (7, 0.000000)\n", + "NX: (11, 0.000000), CG: (11, 0.000000)\n", + "NX: (12, 0.000000), CG: (12, 0.000000)\n", + "NX: (14, 0.000000), CG: (14, 0.000000)\n", + "NX: (15, 0.000000), CG: (15, 0.000000)\n", + "NX: (16, 0.000000), CG: (16, 0.000000)\n", + "NX: (17, 0.000000), CG: (17, 0.000000)\n", + "NX: (18, 0.000000), CG: (18, 0.000000)\n", + "NX: (20, 0.000000), CG: (20, 0.000000)\n", + "NX: (21, 0.000000), CG: (21, 0.000000)\n", + "NX: (22, 0.000000), CG: (22, 0.000000)\n", + "NX: (26, 0.000000), CG: (26, 0.000000)\n" + ] + } + ], + "source": [ + "# The lists contain tuples of (vertex ID, betweenness_centrality score),\n", + "# sorted based on the score.\n", + "nx_sorted = sorted(karate_nx_bc_results.items(), key=lambda t:t[1], reverse=True)\n", + "cg_sorted = sorted(karate_cg_bc_results.items(), key=lambda t:t[1], reverse=True)\n", + "\n", + "for i in range(len(nx_sorted)):\n", + " print(\"NX: (%d, %.6f), CG: (%d, %.6f)\" % (nx_sorted[i] + cg_sorted[i]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Here we can see that the results match exactly as expected. \n", + "\n", + "For larger graphs, results are harder to compare given that `betweenness_centrality` is an approximation algorithm influenced by the random selection of paths used to compute the betweenness_centrality score of each vertex. The argument `k` is used for limiting the number of paths used in the computation, since using every path for every vertex would be prohibitively expensive for large graphs. For small graphs, `k` need not be specified, which allows `betweenness_centrality` to use all paths for all vertices and makes for an easier comparison." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### `betweenness_centrality` on larger graphs - The U.S. Patent Citation Network1\n", + "\n", + "The U.S. Patent Citation Network dataset is much larger with over 3.7M nodes and over 16.5M edges and demonstrates how `nx-cugraph` enables NetworkX to run `betweenness_centrality` on graphs this large (and larger) in seconds instead of minutes.\n", + "\n", + "#### NetworkX default implementation" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "downloading https://snap.stanford.edu/data/cit-Patents.txt.gz...done\n", + "unzipping cit-Patents.txt.gz...done\n", + "reading csv to dataframe...done\n", + "creating NX graph from dataframe...done\n" + ] + } + ], + "source": [ + "import os\n", + "# Unset NETWORKX_AUTOMATIC_BACKENDS so the default NetworkX implementation is used\n", + "os.environ.pop(\"NETWORKX_AUTOMATIC_BACKENDS\", None)\n", + "nx = reimport_networkx()\n", + "# Create the cit-Patents graph - this will also download the dataset if not previously downloaded\n", + "cit_patents_graph = create_cit_patents_graph()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Since this is a large graph, a k value must be set so the computation returns in a reasonable time\n", + "k = 40" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Because this run will take time, `%%timeit` is restricted to a single pass.\n", + "\n", + "*NOTE: this run may take approximately 1 minute*" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1min 4s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit -r 1\n", + "results = nx.betweenness_centrality(cit_patents_graph, k=k)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Something to note is that `%%timeit` disables garbage collection by default, which may not be something a user is able to do. To see a more realistic real-world run time, `gc` can be enabled." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# import and run the garbage collector upfront prior to using it in the benchmark\n", + "import gc\n", + "gc.collect()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "*NOTE: this run may take approximately 7 minutes!*" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6min 50s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit -r 1 gc.enable()\n", + "nx.betweenness_centrality(cit_patents_graph, k=k)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### `nx-cugraph`\n", + "\n", + "Running on a GPU using `nx-cugraph` can result in a tremendous speedup, especially when graphs reach sizes larger than a few thousand nodes or `k` values become larger to increase accuracy.\n", + "\n", + "Rather than setting the `NETWORKX_AUTOMATIC_BACKENDS` environment variable and re-importing again, this example will demonstrate the `backend=` keyword argument to explicitly direct the NetworkX dispatcher to use the `cugraph` backend." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10.1 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit -r 1 gc.enable()\n", + "nx.betweenness_centrality(cit_patents_graph, k=k, backend=\"cugraph\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "k = 150" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11.6 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit -r 1 gc.enable()\n", + "nx.betweenness_centrality(cit_patents_graph, k=k, backend=\"cugraph\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "For the same graph and the same `k` value, the `\"cugraph\"` backend returns results in seconds instead of minutes. Increasing the `k` value has very little relative impact to runtime due to the high parallel processing ability of the GPU, allowing the user to get improved accuracy for virtually no additional cost." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Type-based dispatching\n", + "\n", + "NetworkX also supports automatically dispatching to backends associated with specific graph types. This requires the user to write code for a specific backend, and therefore requires the backend to be installed, but has the advantage of ensuring a particular behavior without the potential for runtime conversions.\n", + "\n", + "To use type-based dispatching with `nx-cugraph`, the user must import the backend directly in their code to access the utilities provided to create a Graph instance specifically for the `nx-cugraph` backend." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "import nx_cugraph as nxcg" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "The `from_networkx()` API will copy the data from the NetworkX graph instance to the GPU and return a new `nx-cugraph` graph instance. By passing an explicit `nx-cugraph` graph, the NetworkX dispatcher will automatically call the `\"cugraph\"` backend (and only the `\"cugraph\"` backend) without requiring future conversions to copy data to the GPU." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7.92 s ± 2.85 ms per loop (mean ± std. dev. of 2 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit -r 2 global nxcg_cit_patents_graph\n", + "nxcg_cit_patents_graph = nxcg.from_networkx(cit_patents_graph)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.14 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit -r 1 gc.enable()\n", + "nx.betweenness_centrality(nxcg_cit_patents_graph, k=k)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "This notebook demonstrated `nx-cugraph`'s support for `betweenness_centrality`. At the time of this writing, `nx-cugraph` also provides support for `edge_netweenness_centrality` and `louvain_communities`. Other algorithms are scheduled to be supported based on their availability in the cuGraph [pylibcugraph](https://github.com/rapidsai/cugraph/tree/branch-23.10/python/pylibcugraph/pylibcugraph) package and demand by the NetworkX community.\n", + "\n", + "#### Benchmark Results\n", + "The results included in this notebook were generated on a workstation with the following hardware:\n", + "\n", + "\n", + " \n", + " \n", + "
CPU:Intel(R) Xeon(R) Gold 6128 CPU @ 3.40GHz, 45GB
GPU:Quatro RTX 8000, 50GB
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "1 Information on the U.S. Patent Citation Network dataset used in this notebook is as follows:\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
Authors:Jure Leskovec and Andrej Krevl
Title:SNAP Datasets, Stanford Large Network Dataset Collection
URL:http://snap.stanford.edu/data
Date:June 2014
\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index 138d384ebcf..b73ccb0cf9a 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -10,11 +10,11 @@ channels: - conda-forge - nvidia dependencies: -- cugraph==23.10.* +- cugraph==23.12.* - dgl>=1.1.0.cu* - pandas - pre-commit -- pylibcugraphops==23.10.* +- pylibcugraphops==23.12.* - pytest - pytest-benchmark - pytest-cov diff --git a/python/cugraph-dgl/cugraph_dgl/VERSION b/python/cugraph-dgl/cugraph_dgl/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/cugraph-dgl/cugraph_dgl/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/cugraph-dgl/cugraph_dgl/__init__.py b/python/cugraph-dgl/cugraph_dgl/__init__.py index b30cd6c79d9..03ff50896a4 100644 --- a/python/cugraph-dgl/cugraph_dgl/__init__.py +++ b/python/cugraph-dgl/cugraph_dgl/__init__.py @@ -20,4 +20,4 @@ import cugraph_dgl.dataloading import cugraph_dgl.nn -__version__ = "23.10.00" +from cugraph_dgl._version import __git_commit__, __version__ diff --git a/python/cugraph-dgl/cugraph_dgl/_version.py b/python/cugraph-dgl/cugraph_dgl/_version.py new file mode 100644 index 00000000000..f95a4705467 --- /dev/null +++ b/python/cugraph-dgl/cugraph_dgl/_version.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("cugraph_dgl").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/cugraph-dgl/cugraph_dgl/dataloading/utils/sampling_helpers.py b/python/cugraph-dgl/cugraph_dgl/dataloading/utils/sampling_helpers.py index a4f64668348..f674bece8be 100644 --- a/python/cugraph-dgl/cugraph_dgl/dataloading/utils/sampling_helpers.py +++ b/python/cugraph-dgl/cugraph_dgl/dataloading/utils/sampling_helpers.py @@ -14,7 +14,6 @@ from typing import List, Tuple, Dict, Optional from collections import defaultdict import cudf -import cupy from cugraph.utilities.utils import import_optional from cugraph_dgl.nn import SparseGraph @@ -444,53 +443,58 @@ def _process_sampled_df_csc( destinations, respectively. """ # dropna - major_offsets = df.major_offsets.dropna().values - label_hop_offsets = df.label_hop_offsets.dropna().values - renumber_map_offsets = df.renumber_map_offsets.dropna().values - renumber_map = df.map.dropna().values - minors = df.minors.dropna().values + major_offsets = cast_to_tensor(df.major_offsets.dropna()) + label_hop_offsets = cast_to_tensor(df.label_hop_offsets.dropna()) + renumber_map_offsets = cast_to_tensor(df.renumber_map_offsets.dropna()) + renumber_map = cast_to_tensor(df.map.dropna()) + minors = cast_to_tensor(df.minors.dropna()) - n_batches = renumber_map_offsets.size - 1 - n_hops = int((label_hop_offsets.size - 1) / n_batches) + n_batches = len(renumber_map_offsets) - 1 + n_hops = int((len(label_hop_offsets) - 1) / n_batches) # make global offsets local - major_offsets -= major_offsets[0] - label_hop_offsets -= label_hop_offsets[0] - renumber_map_offsets -= renumber_map_offsets[0] + # Have to make a clone as pytorch does not allow + # in-place operations on tensors + major_offsets -= major_offsets[0].clone() + label_hop_offsets -= label_hop_offsets[0].clone() + renumber_map_offsets -= renumber_map_offsets[0].clone() # get the sizes of each adjacency matrix (for MFGs) mfg_sizes = (label_hop_offsets[1:] - label_hop_offsets[:-1]).reshape( (n_batches, n_hops) ) n_nodes = renumber_map_offsets[1:] - renumber_map_offsets[:-1] - mfg_sizes = cupy.hstack((mfg_sizes, n_nodes.reshape(n_batches, -1))) + mfg_sizes = torch.hstack((mfg_sizes, n_nodes.reshape(n_batches, -1))) if reverse_hop_id: - mfg_sizes = mfg_sizes[:, ::-1] + mfg_sizes = mfg_sizes.flip(1) tensors_dict = {} renumber_map_list = [] + # Note: minors and major_offsets from BulkSampler are of type int32 + # and int64 respectively. Since pylibcugraphops binding code doesn't + # support distinct node and edge index type, we simply casting both + # to int32 for now. + minors = minors.int() + major_offsets = major_offsets.int() + # Note: We transfer tensors to CPU here to avoid the overhead of + # transferring them in each iteration of the for loop below. + major_offsets_cpu = major_offsets.to("cpu").numpy() + label_hop_offsets_cpu = label_hop_offsets.to("cpu").numpy() + for batch_id in range(n_batches): batch_dict = {} - for hop_id in range(n_hops): hop_dict = {} idx = batch_id * n_hops + hop_id # idx in label_hop_offsets - major_offsets_start = label_hop_offsets[idx].item() - major_offsets_end = label_hop_offsets[idx + 1].item() - minors_start = major_offsets[major_offsets_start].item() - minors_end = major_offsets[major_offsets_end].item() - # Note: minors and major_offsets from BulkSampler are of type int32 - # and int64 respectively. Since pylibcugraphops binding code doesn't - # support distinct node and edge index type, we simply casting both - # to int32 for now. - hop_dict["minors"] = torch.as_tensor( - minors[minors_start:minors_end], device="cuda" - ).int() - hop_dict["major_offsets"] = torch.as_tensor( + major_offsets_start = label_hop_offsets_cpu[idx] + major_offsets_end = label_hop_offsets_cpu[idx + 1] + minors_start = major_offsets_cpu[major_offsets_start] + minors_end = major_offsets_cpu[major_offsets_end] + hop_dict["minors"] = minors[minors_start:minors_end] + hop_dict["major_offsets"] = ( major_offsets[major_offsets_start : major_offsets_end + 1] - - major_offsets[major_offsets_start], - device="cuda", - ).int() + - major_offsets[major_offsets_start] + ) if reverse_hop_id: batch_dict[n_hops - 1 - hop_id] = hop_dict else: @@ -499,12 +503,9 @@ def _process_sampled_df_csc( tensors_dict[batch_id] = batch_dict renumber_map_list.append( - torch.as_tensor( - renumber_map[ - renumber_map_offsets[batch_id] : renumber_map_offsets[batch_id + 1] - ], - device="cuda", - ) + renumber_map[ + renumber_map_offsets[batch_id] : renumber_map_offsets[batch_id + 1] + ], ) return tensors_dict, renumber_map_list, mfg_sizes.tolist() diff --git a/python/cugraph-dgl/examples/graphsage/node-classification.py b/python/cugraph-dgl/examples/graphsage/node-classification.py index 320890b0312..539fd86d136 100644 --- a/python/cugraph-dgl/examples/graphsage/node-classification.py +++ b/python/cugraph-dgl/examples/graphsage/node-classification.py @@ -243,7 +243,9 @@ def train(args, device, g, dataset, model): else: g = g.to("cuda" if args.mode == "gpu_dgl" else "cpu") - device = torch.device("cpu" if args.mode == "cpu" else "cuda") + device = torch.device( + "cpu" if args.mode == "cpu" or args.mode == "mixed" else "cuda" + ) # create GraphSAGE model feat_shape = ( diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 50354184133..eff7a20f0aa 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "cugraph-dgl" -version = "23.10.00" +dynamic = ["version"] description = "cugraph extensions for DGL" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -19,7 +19,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cugraph==23.10.*", + "cugraph==23.12.*", "numba>=0.57", "numpy>=1.21", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -35,6 +35,9 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [tool.setuptools] license-files = ["LICENSE"] +[tool.setuptools.dynamic] +version = {file = "cugraph_dgl/VERSION"} + [tool.setuptools.packages.find] include = [ "cugraph_dgl*", diff --git a/python/cugraph-dgl/setup.py b/python/cugraph-dgl/setup.py index 6991b23b0fb..afb8002af42 100644 --- a/python/cugraph-dgl/setup.py +++ b/python/cugraph-dgl/setup.py @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup +from setuptools import find_packages, setup -setup() +packages = find_packages(include=["cugraph_dgl*"]) +setup( + package_data={key: ["VERSION"] for key in packages}, +) diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index 4e5159e6b45..71d1c7e389c 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -10,16 +10,16 @@ channels: - conda-forge - nvidia dependencies: -- cugraph==23.10.* +- cugraph==23.12.* - pandas - pre-commit -- pyg=2.3.1=*torch_2.0.0*cu118* -- pylibcugraphops==23.10.* +- pyg>=2.4.0 +- pylibcugraphops==23.12.* - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda==11.8 -- pytorch==2.0 +- pytorch>=2.0 - scipy name: cugraph_pyg_dev_cuda-118 diff --git a/python/cugraph-pyg/cugraph_pyg/VERSION b/python/cugraph-pyg/cugraph_pyg/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/cugraph-pyg/cugraph_pyg/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/cugraph-pyg/cugraph_pyg/__init__.py b/python/cugraph-pyg/cugraph_pyg/__init__.py index f8187059b86..719751c966a 100644 --- a/python/cugraph-pyg/cugraph_pyg/__init__.py +++ b/python/cugraph-pyg/cugraph_pyg/__init__.py @@ -11,4 +11,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "23.10.00" +from cugraph_pyg._version import __git_commit__, __version__ diff --git a/python/cugraph-pyg/cugraph_pyg/_version.py b/python/cugraph-pyg/cugraph_pyg/_version.py new file mode 100644 index 00000000000..963052da909 --- /dev/null +++ b/python/cugraph-pyg/cugraph_pyg/_version.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("cugraph_pyg").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index fd2172e6ade..14dc5d84f90 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -27,11 +27,12 @@ import cugraph import warnings -from cugraph.utilities.utils import import_optional, MissingModule +import dask.array as dar +import dask.dataframe as dd +import dask.distributed as distributed +import dask_cudf -dd = import_optional("dask.dataframe") -distributed = import_optional("dask.distributed") -dask_cudf = import_optional("dask_cudf") +from cugraph.utilities.utils import import_optional, MissingModule torch = import_optional("torch") torch_geometric = import_optional("torch_geometric") @@ -210,11 +211,14 @@ class EXPERIMENTAL__CuGraphStore: def __init__( self, F: cugraph.gnn.FeatureStore, - G: Union[Dict[str, Tuple[TensorType]], Dict[str, int]], + G: Union[ + Dict[Tuple[str, str, str], Tuple[TensorType]], + Dict[Tuple[str, str, str], int], + ], num_nodes_dict: Dict[str, int], *, multi_gpu: bool = False, - order: str = "CSC", + order: str = "CSR", ): """ Constructs a new CuGraphStore from the provided @@ -260,11 +264,11 @@ def __init__( Whether the store should be backed by a multi-GPU graph. Requires dask to have been set up. - order: str (Optional ["CSR", "CSC"], default = CSC) - The order to use for sampling. Should nearly always be CSC - unless there is a specific expectation of "reverse" sampling. - It is also not uncommon to use CSR order for correctness - testing, which some cuGraph-PyG tests do. + order: str (Optional ["CSR", "CSC"], default = CSR) + The order to use for sampling. CSR corresponds to the + standard OGB dataset order that is usually used in PyG. + CSC order constructs the same graph as CSR, but with + edges in the opposite direction. """ if None in G: @@ -320,7 +324,13 @@ def __init__( def __del__(self): if self.__is_graph_owner: if isinstance(self.__graph._plc_graph, dict): - distributed.get_client().unpublish_dataset("cugraph_graph") + try: + distributed.get_client().unpublish_dataset("cugraph_graph") + except TypeError: + warnings.warn( + "Could not unpublish graph dataset, most likely because" + " dask has already shut down." + ) del self.__graph def __make_offsets(self, input_dict): @@ -358,6 +368,13 @@ def __infer_offsets( } ) + def __dask_array_from_numpy(self, array: np.ndarray, npartitions: int): + return dar.from_array( + array, + meta=np.array([], dtype=array.dtype), + chunks=max(1, len(array) // npartitions), + ) + def __construct_graph( self, edge_info: Dict[Tuple[str, str, str], List[TensorType]], @@ -455,22 +472,32 @@ def __construct_graph( ] ) - df = pandas.DataFrame( - { - "src": pandas.Series(na_dst) - if order == "CSC" - else pandas.Series(na_src), - "dst": pandas.Series(na_src) - if order == "CSC" - else pandas.Series(na_dst), - "etp": pandas.Series(na_etp), - } - ) - vertex_dtype = df.src.dtype + vertex_dtype = na_src.dtype if multi_gpu: - nworkers = len(distributed.get_client().scheduler_info()["workers"]) - df = dd.from_pandas(df, npartitions=nworkers if len(df) > 32 else 1) + client = distributed.get_client() + nworkers = len(client.scheduler_info()["workers"]) + npartitions = nworkers * 4 + + src_dar = self.__dask_array_from_numpy(na_src, npartitions) + del na_src + + dst_dar = self.__dask_array_from_numpy(na_dst, npartitions) + del na_dst + + etp_dar = self.__dask_array_from_numpy(na_etp, npartitions) + del na_etp + + df = dd.from_dask_array(etp_dar, columns=["etp"]) + df["src"] = dst_dar if order == "CSC" else src_dar + df["dst"] = src_dar if order == "CSC" else dst_dar + + del src_dar + del dst_dar + del etp_dar + + if df.etp.dtype != "int32": + raise ValueError("Edge type must be int32!") # Ensure the dataframe is constructed on each partition # instead of adding additional synchronization head from potential @@ -478,9 +505,9 @@ def __construct_graph( def get_empty_df(): return cudf.DataFrame( { + "etp": cudf.Series([], dtype="int32"), "src": cudf.Series([], dtype=vertex_dtype), "dst": cudf.Series([], dtype=vertex_dtype), - "etp": cudf.Series([], dtype="int32"), } ) @@ -491,9 +518,23 @@ def get_empty_df(): if len(f) > 0 else get_empty_df(), meta=get_empty_df(), - ).reset_index(drop=True) + ).reset_index( + drop=True + ) # should be ok for dask else: - df = cudf.from_pandas(df).reset_index(drop=True) + df = pandas.DataFrame( + { + "src": pandas.Series(na_dst) + if order == "CSC" + else pandas.Series(na_src), + "dst": pandas.Series(na_src) + if order == "CSC" + else pandas.Series(na_dst), + "etp": pandas.Series(na_etp), + } + ) + df = cudf.from_pandas(df) + df.reset_index(drop=True, inplace=True) graph = cugraph.MultiGraph(directed=True) if multi_gpu: @@ -512,6 +553,7 @@ def get_empty_df(): edge_type="etp", ) + del df return graph @property @@ -738,7 +780,7 @@ def _subgraph(self, edge_types: List[tuple] = None) -> cugraph.MultiGraph: def _get_vertex_groups_from_sample( self, nodes_of_interest: TensorType, is_sorted: bool = False - ) -> dict: + ) -> Dict[str, torch.Tensor]: """ Given a tensor of nodes of interest, this method a single dictionary, noi_index. @@ -802,7 +844,10 @@ def _get_sample_from_vertex_groups( def _get_renumbered_edge_groups_from_sample( self, sampling_results: cudf.DataFrame, noi_index: dict - ) -> Tuple[dict, dict]: + ) -> Tuple[ + Dict[Tuple[str, str, str], torch.Tensor], + Tuple[Dict[Tuple[str, str, str], torch.Tensor]], + ]: """ Given a cudf (NOT dask_cudf) DataFrame of sampling results and a dictionary of non-renumbered vertex ids grouped by vertex type, this method diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index 8552e7412e0..200a82b460b 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -15,6 +15,7 @@ import os import re +import warnings import cupy import cudf @@ -51,7 +52,9 @@ def __init__( graph_store: CuGraphStore, input_nodes: InputNodes = None, batch_size: int = 0, + *, shuffle: bool = False, + drop_last: bool = True, edge_types: Sequence[Tuple[str]] = None, directory: Union[str, tempfile.TemporaryDirectory] = None, input_files: List[str] = None, @@ -159,23 +162,34 @@ def __init__( if batch_size is None or batch_size < 1: raise ValueError("Batch size must be >= 1") - self.__directory = tempfile.TemporaryDirectory(dir=directory) + self.__directory = ( + tempfile.TemporaryDirectory() if directory is None else directory + ) if isinstance(num_neighbors, dict): raise ValueError("num_neighbors dict is currently unsupported!") - renumber = ( - True - if ( - (len(self.__graph_store.node_types) == 1) - and (len(self.__graph_store.edge_types) == 1) + if "renumber" in kwargs: + warnings.warn( + "Setting renumbering manually could result in invalid output," + " please ensure you intended to do this." + ) + renumber = kwargs.pop("renumber") + else: + renumber = ( + True + if ( + (len(self.__graph_store.node_types) == 1) + and (len(self.__graph_store.edge_types) == 1) + ) + else False ) - else False - ) bulk_sampler = BulkSampler( batch_size, - self.__directory.name, + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name, self.__graph_store._subgraph(edge_types), fanout_vals=num_neighbors, with_replacement=replace, @@ -197,29 +211,40 @@ def __init__( # Truncate if we can't evenly divide the input array stop = (len(input_nodes) // batch_size) * batch_size - input_nodes = input_nodes[:stop] + input_nodes, remainder = cupy.array_split(input_nodes, [stop]) # Split into batches - input_nodes = cupy.split(input_nodes, len(input_nodes) // batch_size) + input_nodes = cupy.split(input_nodes, max(len(input_nodes) // batch_size, 1)) + + if not drop_last: + input_nodes.append(remainder) self.__num_batches = 0 for batch_num, batch_i in enumerate(input_nodes): - self.__num_batches += 1 - bulk_sampler.add_batches( - cudf.DataFrame( - { - "start": batch_i, - "batch": cupy.full( - batch_size, batch_num + starting_batch_id, dtype="int32" - ), - } - ), - start_col_name="start", - batch_col_name="batch", - ) + batch_len = len(batch_i) + if batch_len > 0: + self.__num_batches += 1 + bulk_sampler.add_batches( + cudf.DataFrame( + { + "start": batch_i, + "batch": cupy.full( + batch_len, batch_num + starting_batch_id, dtype="int32" + ), + } + ), + start_col_name="start", + batch_col_name="batch", + ) bulk_sampler.flush() - self.__input_files = iter(os.listdir(self.__directory.name)) + self.__input_files = iter( + os.listdir( + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name + ) + ) def __next__(self): from time import perf_counter @@ -423,9 +448,6 @@ def __next__(self): sampler_output.edge, ) else: - if self.__graph_store.order == "CSR": - raise ValueError("CSR format incompatible with CSC output") - out = filter_cugraph_store_csc( self.__feature_store, self.__graph_store, @@ -437,11 +459,8 @@ def __next__(self): # Account for CSR format in cuGraph vs. CSC format in PyG if self.__coo and self.__graph_store.order == "CSC": - for node_type in out.edge_index_dict: - out[node_type].edge_index[0], out[node_type].edge_index[1] = ( - out[node_type].edge_index[1], - out[node_type].edge_index[0], - ) + for edge_type in out.edge_index_dict: + out[edge_type].edge_index = out[edge_type].edge_index.flip(dims=[0]) out.set_value_dict("num_sampled_nodes", sampler_output.num_sampled_nodes) out.set_value_dict("num_sampled_edges", sampler_output.num_sampled_edges) diff --git a/python/cugraph-pyg/cugraph_pyg/tests/conftest.py b/python/cugraph-pyg/cugraph_pyg/tests/conftest.py index 083c4a2b37b..1512901822a 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/conftest.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/conftest.py @@ -24,7 +24,7 @@ import torch import numpy as np from cugraph.gnn import FeatureStore -from cugraph.experimental.datasets import karate +from cugraph.datasets import karate import tempfile diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py index 55aebf305da..f5035a38621 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py @@ -15,7 +15,6 @@ from cugraph_pyg.loader import CuGraphNeighborLoader from cugraph_pyg.data import CuGraphStore - from cugraph.utilities.utils import import_optional, MissingModule torch = import_optional("torch") diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py index ed7f70034e2..be8f8245807 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py @@ -120,7 +120,7 @@ def test_get_edge_index(graph, edge_index_type, dask_client): G[et][0] = dask_cudf.from_cudf(cudf.Series(G[et][0]), npartitions=1) G[et][1] = dask_cudf.from_cudf(cudf.Series(G[et][1]), npartitions=1) - cugraph_store = CuGraphStore(F, G, N, multi_gpu=True) + cugraph_store = CuGraphStore(F, G, N, order="CSC", multi_gpu=True) for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( @@ -386,3 +386,29 @@ def test_mg_frame_handle(graph, dask_client): F, G, N = graph cugraph_store = CuGraphStore(F, G, N, multi_gpu=True) assert isinstance(cugraph_store._EXPERIMENTAL__CuGraphStore__graph._plc_graph, dict) + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +def test_cugraph_loader_large_index(dask_client): + large_index = ( + np.random.randint(0, 1_000_000, (100_000_000,)), + np.random.randint(0, 1_000_000, (100_000_000,)), + ) + + large_features = np.random.randint(0, 50, (1_000_000,)) + F = cugraph.gnn.FeatureStore(backend="torch") + F.add_data(large_features, "N", "f") + + store = CuGraphStore( + F, + {("N", "e", "N"): large_index}, + {"N": 1_000_000}, + multi_gpu=True, + ) + + graph = store._subgraph() + assert isinstance(graph, cugraph.Graph) + + el = graph.view_edge_list().compute() + assert (el["src"].values_host - large_index[0]).sum() == 0 + assert (el["dst"].values_host - large_index[1]).sum() == 0 diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py index 03274948158..9813fa933ee 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py @@ -18,6 +18,7 @@ import cudf import cupy +import numpy as np from cugraph_pyg.loader import CuGraphNeighborLoader from cugraph_pyg.loader import BulkSampleLoader @@ -27,13 +28,26 @@ from cugraph.gnn import FeatureStore from cugraph.utilities.utils import import_optional, MissingModule +from typing import Dict, Tuple + torch = import_optional("torch") torch_geometric = import_optional("torch_geometric") trim_to_layer = import_optional("torch_geometric.utils.trim_to_layer") +try: + import torch_sparse # noqa: F401 + + HAS_TORCH_SPARSE = True +except: # noqa: E722 + HAS_TORCH_SPARSE = False + @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_basic(karate_gnn): +def test_cugraph_loader_basic( + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ] +): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -59,7 +73,11 @@ def test_cugraph_loader_basic(karate_gnn): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_hetero(karate_gnn): +def test_cugraph_loader_hetero( + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ] +): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -200,6 +218,7 @@ def test_cugraph_loader_from_disk_subset(): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.skipif(not HAS_TORCH_SPARSE, reason="torch-sparse not available") def test_cugraph_loader_from_disk_subset_csr(): m = [2, 9, 99, 82, 11, 13] n = torch.arange(1, 1 + len(m), dtype=torch.int32) @@ -332,8 +351,9 @@ def test_cugraph_loader_e2e_coo(): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.skipif(not HAS_TORCH_SPARSE, reason="torch-sparse not available") @pytest.mark.parametrize("framework", ["pyg", "cugraph-ops"]) -def test_cugraph_loader_e2e_csc(framework): +def test_cugraph_loader_e2e_csc(framework: str): m = [2, 9, 99, 82, 9, 3, 18, 1, 12] x = torch.randint(3000, (256, 256)).to(torch.float32) F = FeatureStore() @@ -433,3 +453,78 @@ def test_cugraph_loader_e2e_csc(framework): x = x.narrow(dim=0, start=0, length=s - num_sampled_nodes[1]) assert list(x.shape) == [1, 1] + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.parametrize("drop_last", [True, False]) +def test_drop_last(drop_last): + N = {"N": 10} + G = { + ("N", "e", "N"): torch.stack( + [torch.tensor([0, 1, 2, 3, 4]), torch.tensor([5, 6, 7, 8, 9])] + ) + } + F = FeatureStore(backend="torch") + F.add_data(torch.arange(10), "N", "z") + + store = CuGraphStore(F, G, N) + with tempfile.TemporaryDirectory() as dir: + loader = CuGraphNeighborLoader( + (store, store), + input_nodes=torch.tensor([0, 1, 2, 3, 4]), + num_neighbors=[1], + batch_size=2, + shuffle=False, + drop_last=drop_last, + batches_per_partition=1, + directory=dir, + ) + + t = torch.tensor([]) + for batch in loader: + t = torch.concat([t, batch["N"].z]) + + t = t.tolist() + + files = os.listdir(dir) + assert len(files) == 2 if drop_last else 3 + assert "batch=0-0.parquet" in files + assert "batch=1-1.parquet" in files + if not drop_last: + assert "batch=2-2.parquet" in files + + +@pytest.mark.parametrize("directory", ["local", "temp"]) +def test_load_directory( + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ], + directory: str, +): + if directory == "local": + local_dir = tempfile.TemporaryDirectory(dir=".") + + cugraph_store = CuGraphStore(*karate_gnn) + cugraph_loader = CuGraphNeighborLoader( + (cugraph_store, cugraph_store), + torch.arange(8, dtype=torch.int64), + 2, + num_neighbors=[8, 4, 2], + random_state=62, + replace=False, + directory=None if directory == "temp" else local_dir.name, + batches_per_partition=1, + ) + + it = iter(cugraph_loader) + next_batch = next(it) + assert next_batch is not None + + if directory == "local": + assert len(os.listdir(local_dir.name)) == 4 + + count = 1 + while next(it, None) is not None: + count += 1 + + assert count == 4 diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py index da3043760d4..b39ebad8254 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py @@ -113,7 +113,7 @@ def test_get_edge_index(graph, edge_index_type): G[et][0] = cudf.Series(G[et][0]) G[et][1] = cudf.Series(G[et][1]) - cugraph_store = CuGraphStore(F, G, N) + cugraph_store = CuGraphStore(F, G, N, order="CSC") for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index 218c09fbd1d..95b1fa27402 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -12,7 +12,7 @@ testpaths = ["cugraph_pyg/tests"] [project] name = "cugraph_pyg" -version = "23.10.00" +dynamic = ["version"] description = "cugraph_pyg - PyG support for cuGraph massive-scale, ultra-fast GPU graph analytics." authors = [ { name = "NVIDIA Corporation" }, @@ -26,7 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", ] dependencies = [ - "cugraph==23.10.*", + "cugraph==23.12.*", "numba>=0.57", "numpy>=1.21", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -38,6 +38,9 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [tool.setuptools] license-files = ["LICENSE"] +[tool.setuptools.dynamic] +version = {file = "cugraph_pyg/VERSION"} + [tool.setuptools.packages.find] include = [ "cugraph_pyg*", diff --git a/python/cugraph-pyg/setup.py b/python/cugraph-pyg/setup.py index 1f7db1d3772..50f023050bf 100644 --- a/python/cugraph-pyg/setup.py +++ b/python/cugraph-pyg/setup.py @@ -14,7 +14,7 @@ import os import shutil -from setuptools import Command, setup +from setuptools import Command, find_packages, setup from setuputils import get_environment_option @@ -59,6 +59,8 @@ def run(self): os.system("rm -rf *.egg-info") +packages = find_packages(include=["cugraph_pyg*"]) setup( cmdclass={"clean": CleanCommand}, + package_data={key: ["VERSION"] for key in packages}, ) diff --git a/python/cugraph-service/client/cugraph_service_client/VERSION b/python/cugraph-service/client/cugraph_service_client/VERSION new file mode 120000 index 00000000000..a4e948506b8 --- /dev/null +++ b/python/cugraph-service/client/cugraph_service_client/VERSION @@ -0,0 +1 @@ +../../../../VERSION \ No newline at end of file diff --git a/python/cugraph-service/client/cugraph_service_client/__init__.py b/python/cugraph-service/client/cugraph_service_client/__init__.py index 229d07b8bc6..a9a96ae6c16 100644 --- a/python/cugraph-service/client/cugraph_service_client/__init__.py +++ b/python/cugraph-service/client/cugraph_service_client/__init__.py @@ -35,4 +35,4 @@ from cugraph_service_client.client import CugraphServiceClient from cugraph_service_client.remote_graph import RemoteGraph -__version__ = "23.10.00" +from cugraph_service_client._version import __git_commit__, __version__ diff --git a/python/cugraph-service/client/cugraph_service_client/_version.py b/python/cugraph-service/client/cugraph_service_client/_version.py new file mode 100644 index 00000000000..344361973bb --- /dev/null +++ b/python/cugraph-service/client/cugraph_service_client/_version.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("cugraph_service_client") + .joinpath("VERSION") + .read_text() + .strip() +) +__git_commit__ = "" diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index 3b31a5f2e0a..59539693877 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "cugraph-service-client" -version = "23.10.00" +dynamic = ["version"] description = "cuGraph Service client" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -35,6 +35,9 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [tool.setuptools] license-files = ["LICENSE"] +[tool.setuptools.dynamic] +version = {file = "cugraph_service_client/VERSION"} + [tool.setuptools.packages.find] include = [ "cugraph_service_client", diff --git a/python/cugraph-service/client/setup.py b/python/cugraph-service/client/setup.py index 811a12c50b7..61c758cef4a 100644 --- a/python/cugraph-service/client/setup.py +++ b/python/cugraph-service/client/setup.py @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup +from setuptools import find_packages, setup -setup() +packages = find_packages(include=["cugraph_service_client*"]) +setup( + package_data={key: ["VERSION"] for key in packages}, +) diff --git a/python/cugraph-service/server/cugraph_service_server/VERSION b/python/cugraph-service/server/cugraph_service_server/VERSION new file mode 120000 index 00000000000..a4e948506b8 --- /dev/null +++ b/python/cugraph-service/server/cugraph_service_server/VERSION @@ -0,0 +1 @@ +../../../../VERSION \ No newline at end of file diff --git a/python/cugraph-service/server/cugraph_service_server/__init__.py b/python/cugraph-service/server/cugraph_service_server/__init__.py index 017f0990f89..02473f0ea47 100644 --- a/python/cugraph-service/server/cugraph_service_server/__init__.py +++ b/python/cugraph-service/server/cugraph_service_server/__init__.py @@ -61,4 +61,4 @@ def start_server_blocking( server.serve() # blocks until Ctrl-C (kill -2) -__version__ = "23.10.00" +from cugraph_service_server._version import __git_commit__, __version__ diff --git a/python/cugraph-service/server/cugraph_service_server/_version.py b/python/cugraph-service/server/cugraph_service_server/_version.py new file mode 100644 index 00000000000..7da31f78767 --- /dev/null +++ b/python/cugraph-service/server/cugraph_service_server/_version.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("cugraph_service_server") + .joinpath("VERSION") + .read_text() + .strip() +) +__git_commit__ = "" diff --git a/python/cugraph-service/server/cugraph_service_server/testing/benchmark_server_extension.py b/python/cugraph-service/server/cugraph_service_server/testing/benchmark_server_extension.py index 5f9eac6b2a3..361226c8071 100644 --- a/python/cugraph-service/server/cugraph_service_server/testing/benchmark_server_extension.py +++ b/python/cugraph-service/server/cugraph_service_server/testing/benchmark_server_extension.py @@ -17,7 +17,7 @@ import cugraph from cugraph.experimental import PropertyGraph, MGPropertyGraph -from cugraph.experimental import datasets +from cugraph import datasets from cugraph.generators import rmat diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index 8787cb838be..d68f8055ded 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "cugraph-service-server" -version = "23.10.00" +dynamic = ["version", "entry-points"] description = "cuGraph Service server" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -19,19 +19,18 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cudf==23.10.*", - "cugraph-service-client==23.10.*", - "cugraph==23.10.*", + "cudf==23.12.*", + "cugraph-service-client==23.12.*", + "cugraph==23.12.*", "cupy-cuda11x>=12.0.0", - "dask-cuda==23.10.*", - "dask-cudf==23.10.*", - "dask==2023.9.2", - "distributed==2023.9.2", + "dask-cuda==23.12.*", + "dask-cudf==23.12.*", "numba>=0.57", "numpy>=1.21", - "rmm==23.10.*", + "rapids-dask-dependency==23.12.*", + "rmm==23.12.*", "thriftpy2", - "ucx-py==0.34.*", + "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -39,7 +38,6 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ] -dynamic = ["entry-points"] [project.optional-dependencies] test = [ @@ -62,6 +60,9 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [tool.setuptools] license-files = ["LICENSE"] +[tool.setuptools.dynamic] +version = {file = "cugraph_service_server/VERSION"} + [tool.setuptools.packages.find] include = [ "cugraph_service_server", diff --git a/python/cugraph-service/server/setup.py b/python/cugraph-service/server/setup.py index 5203b76c659..91864168e2c 100644 --- a/python/cugraph-service/server/setup.py +++ b/python/cugraph-service/server/setup.py @@ -11,12 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup +from setuptools import find_packages, setup +packages = find_packages(include=["cugraph_service_server*"]) setup( entry_points={ "console_scripts": [ "cugraph-service-server=cugraph_service_server.__main__:main" ], }, + package_data={key: ["VERSION"] for key in packages}, ) diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 64db9571dc9..8693c0e9e1f 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -14,7 +14,7 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -set(cugraph_version 23.10.00) +set(cugraph_version 23.12.00) include(../../fetch_rapids.cmake) diff --git a/python/cugraph/cugraph/VERSION b/python/cugraph/cugraph/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/cugraph/cugraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/cugraph/cugraph/__init__.py b/python/cugraph/cugraph/__init__.py index 43cb30beceb..f635d215696 100644 --- a/python/cugraph/cugraph/__init__.py +++ b/python/cugraph/cugraph/__init__.py @@ -120,4 +120,4 @@ from cugraph import exceptions -__version__ = "23.10.00" +from cugraph._version import __git_commit__, __version__ diff --git a/python/cugraph/cugraph/_version.py b/python/cugraph/cugraph/_version.py new file mode 100644 index 00000000000..710afb87e29 --- /dev/null +++ b/python/cugraph/cugraph/_version.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("cugraph").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/cugraph/cugraph/dask/community/egonet.py b/python/cugraph/cugraph/dask/community/egonet.py index 06f5d5b9a79..e49d4777cef 100644 --- a/python/cugraph/cugraph/dask/community/egonet.py +++ b/python/cugraph/cugraph/dask/community/egonet.py @@ -18,7 +18,9 @@ import cugraph.dask.comms.comms as Comms import dask_cudf import cudf -from cugraph.dask.common.input_utils import get_distributed_data +from cugraph.dask.common.part_utils import ( + persist_dask_df_equal_parts_per_worker, +) from pylibcugraph import ResourceHandle, ego_graph as pylibcugraph_ego_graph @@ -135,11 +137,7 @@ def ego_graph(input_graph, n, radius=1, center=True): n = dask_cudf.from_cudf(n, npartitions=min(input_graph._npartitions, len(n))) n = n.astype(n_type) - n = get_distributed_data(n) - wait(n) - - n = n.worker_to_parts - + n = persist_dask_df_equal_parts_per_worker(n, client, return_type="dict") do_expensive_check = False result = [ @@ -147,13 +145,13 @@ def ego_graph(input_graph, n, radius=1, center=True): _call_ego_graph, Comms.get_session_id(), input_graph._plc_graph[w], - n[w][0], + n_[0] if n_ else cudf.Series(dtype=n_type), radius, do_expensive_check, workers=[w], allow_other_workers=False, ) - for w in Comms.get_workers() + for w, n_ in n.items() ] wait(result) diff --git a/python/cugraph/cugraph/dask/community/induced_subgraph.py b/python/cugraph/cugraph/dask/community/induced_subgraph.py index 5d902f667a4..d079bcaf653 100644 --- a/python/cugraph/cugraph/dask/community/induced_subgraph.py +++ b/python/cugraph/cugraph/dask/community/induced_subgraph.py @@ -19,7 +19,9 @@ import dask_cudf import cudf import cupy as cp -from cugraph.dask.common.input_utils import get_distributed_data +from cugraph.dask.common.part_utils import ( + persist_dask_df_equal_parts_per_worker, +) from typing import Union, Tuple from pylibcugraph import ( @@ -154,15 +156,12 @@ def induced_subgraph( vertices_type = input_graph.input_df.dtypes[0] if isinstance(vertices, (cudf.Series, cudf.DataFrame)): - vertices = dask_cudf.from_cudf( - vertices, npartitions=min(input_graph._npartitions, len(vertices)) - ) + vertices = dask_cudf.from_cudf(vertices, npartitions=input_graph._npartitions) vertices = vertices.astype(vertices_type) - vertices = get_distributed_data(vertices) - wait(vertices) - - vertices = vertices.worker_to_parts + vertices = persist_dask_df_equal_parts_per_worker( + vertices, client, return_type="dict" + ) do_expensive_check = False @@ -171,13 +170,13 @@ def induced_subgraph( _call_induced_subgraph, Comms.get_session_id(), input_graph._plc_graph[w], - vertices[w][0], + vertices_[0] if vertices_ else cudf.Series(dtype=vertices_type), offsets, do_expensive_check, workers=[w], allow_other_workers=False, ) - for w in Comms.get_workers() + for w, vertices_ in vertices.items() ] wait(result) diff --git a/python/cugraph/cugraph/dask/community/leiden.py b/python/cugraph/cugraph/dask/community/leiden.py index 75582fa48f7..10a266ed519 100644 --- a/python/cugraph/cugraph/dask/community/leiden.py +++ b/python/cugraph/cugraph/dask/community/leiden.py @@ -125,9 +125,19 @@ def leiden( Examples -------- - >>> from cugraph.experimental.datasets import karate - >>> G = karate.get_graph(fetch=True) - >>> parts, modularity_score = cugraph.leiden(G) + >>> import cugraph.dask as dcg + >>> import dask_cudf + >>> # ... Init a DASK Cluster + >>> # see https://docs.rapids.ai/api/cugraph/stable/dask-cugraph.html + >>> # Download dataset from https://github.com/rapidsai/cugraph/datasets/.. + >>> chunksize = dcg.get_chunksize(datasets_path / "karate.csv") + >>> ddf = dask_cudf.read_csv(datasets_path / "karate.csv", + ... chunksize=chunksize, delimiter=" ", + ... names=["src", "dst", "value"], + ... dtype=["int32", "int32", "float32"]) + >>> dg = cugraph.Graph() + >>> dg.from_dask_cudf_edgelist(ddf, source='src', destination='dst') + >>> parts, modularity_score = dcg.leiden(dg) """ diff --git a/python/cugraph/cugraph/dask/community/louvain.py b/python/cugraph/cugraph/dask/community/louvain.py index 8efbbafaf7b..e83d41811ea 100644 --- a/python/cugraph/cugraph/dask/community/louvain.py +++ b/python/cugraph/cugraph/dask/community/louvain.py @@ -129,9 +129,19 @@ def louvain( Examples -------- - >>> from cugraph.experimental.datasets import karate - >>> G = karate.get_graph(fetch=True) - >>> parts = cugraph.louvain(G) + >>> import cugraph.dask as dcg + >>> import dask_cudf + >>> # ... Init a DASK Cluster + >>> # see https://docs.rapids.ai/api/cugraph/stable/dask-cugraph.html + >>> # Download dataset from https://github.com/rapidsai/cugraph/datasets/.. + >>> chunksize = dcg.get_chunksize(datasets_path / "karate.csv") + >>> ddf = dask_cudf.read_csv(datasets_path / "karate.csv", + ... chunksize=chunksize, delimiter=" ", + ... names=["src", "dst", "value"], + ... dtype=["int32", "int32", "float32"]) + >>> dg = cugraph.Graph() + >>> dg.from_dask_cudf_edgelist(ddf, source='src', destination='dst') + >>> parts, modularity_score = dcg.louvain(dg) """ diff --git a/python/cugraph/cugraph/dask/link_analysis/pagerank.py b/python/cugraph/cugraph/dask/link_analysis/pagerank.py index 2dfd25fa522..1dffb3cba78 100644 --- a/python/cugraph/cugraph/dask/link_analysis/pagerank.py +++ b/python/cugraph/cugraph/dask/link_analysis/pagerank.py @@ -28,7 +28,9 @@ ) import cugraph.dask.comms.comms as Comms -from cugraph.dask.common.input_utils import get_distributed_data +from cugraph.dask.common.part_utils import ( + persist_dask_df_equal_parts_per_worker, +) from cugraph.exceptions import FailedToConvergeError @@ -352,7 +354,14 @@ def pagerank( personalization, npartitions=len(Comms.get_workers()) ) - data_prsztn = get_distributed_data(personalization_ddf) + data_prsztn = persist_dask_df_equal_parts_per_worker( + personalization_ddf, client, return_type="dict" + ) + + empty_df = cudf.DataFrame(columns=list(personalization_ddf.columns)) + empty_df = empty_df.astype( + dict(zip(personalization_ddf.columns, personalization_ddf.dtypes)) + ) result = [ client.submit( @@ -361,7 +370,7 @@ def pagerank( input_graph._plc_graph[w], precomputed_vertex_out_weight_vertices, precomputed_vertex_out_weight_sums, - data_personalization[0], + data_personalization[0] if data_personalization else empty_df, initial_guess_vertices, initial_guess_values, alpha, @@ -372,7 +381,7 @@ def pagerank( workers=[w], allow_other_workers=False, ) - for w, data_personalization in data_prsztn.worker_to_parts.items() + for w, data_personalization in data_prsztn.items() ] else: result = [ diff --git a/python/cugraph/cugraph/dask/sampling/random_walks.py b/python/cugraph/cugraph/dask/sampling/random_walks.py index 993544ac45c..bb9baf2c92c 100644 --- a/python/cugraph/cugraph/dask/sampling/random_walks.py +++ b/python/cugraph/cugraph/dask/sampling/random_walks.py @@ -16,6 +16,9 @@ import dask_cudf import cudf import operator as op +from cugraph.dask.common.part_utils import ( + persist_dask_df_equal_parts_per_worker, +) from pylibcugraph import ResourceHandle @@ -24,7 +27,6 @@ ) from cugraph.dask.comms import comms as Comms -from cugraph.dask.common.input_utils import get_distributed_data def convert_to_cudf(cp_paths, number_map=None, is_vertex_paths=False): @@ -104,7 +106,7 @@ def random_walks( max_path_length : int The maximum path length """ - + client = default_client() if isinstance(start_vertices, int): start_vertices = [start_vertices] @@ -126,23 +128,21 @@ def random_walks( start_vertices, npartitions=min(input_graph._npartitions, len(start_vertices)) ) start_vertices = start_vertices.astype(start_vertices_type) - start_vertices = get_distributed_data(start_vertices) - wait(start_vertices) - start_vertices = start_vertices.worker_to_parts - - client = default_client() + start_vertices = persist_dask_df_equal_parts_per_worker( + start_vertices, client, return_type="dict" + ) result = [ client.submit( _call_plc_uniform_random_walks, Comms.get_session_id(), input_graph._plc_graph[w], - start_vertices[w][0], + start_v[0] if start_v else cudf.Series(dtype=start_vertices_type), max_depth, workers=[w], allow_other_workers=False, ) - for w in Comms.get_workers() + for w, start_v in start_vertices.items() ] wait(result) diff --git a/python/cugraph/cugraph/dask/traversal/bfs.py b/python/cugraph/cugraph/dask/traversal/bfs.py index cf467aaa18f..412fd851ad6 100644 --- a/python/cugraph/cugraph/dask/traversal/bfs.py +++ b/python/cugraph/cugraph/dask/traversal/bfs.py @@ -16,7 +16,9 @@ from pylibcugraph import ResourceHandle, bfs as pylibcugraph_bfs from dask.distributed import wait, default_client -from cugraph.dask.common.input_utils import get_distributed_data +from cugraph.dask.common.part_utils import ( + persist_dask_df_equal_parts_per_worker, +) import cugraph.dask.comms.comms as Comms import cudf import dask_cudf @@ -159,8 +161,13 @@ def bfs(input_graph, start, depth_limit=None, return_distances=True, check_start tmp_col_names = None start = input_graph.lookup_internal_vertex_id(start, tmp_col_names) + vertex_dtype = start.dtype # if the edgelist was renumbered, update + # the vertex type accordingly + + data_start = persist_dask_df_equal_parts_per_worker( + start, client, return_type="dict" + ) - data_start = get_distributed_data(start) do_expensive_check = False # FIXME: Why is 'direction_optimizing' not part of the python cugraph API # and why is it set to 'False' by default @@ -171,7 +178,7 @@ def bfs(input_graph, start, depth_limit=None, return_distances=True, check_start _call_plc_bfs, Comms.get_session_id(), input_graph._plc_graph[w], - st[0], + st[0] if st else cudf.Series(dtype=vertex_dtype), depth_limit, direction_optimizing, return_distances, @@ -179,7 +186,7 @@ def bfs(input_graph, start, depth_limit=None, return_distances=True, check_start workers=[w], allow_other_workers=False, ) - for w, st in data_start.worker_to_parts.items() + for w, st in data_start.items() ] wait(cupy_result) diff --git a/python/cugraph/cugraph/datasets/__init__.py b/python/cugraph/cugraph/datasets/__init__.py index 65a820f108b..ac18274d354 100644 --- a/python/cugraph/cugraph/datasets/__init__.py +++ b/python/cugraph/cugraph/datasets/__init__.py @@ -39,3 +39,13 @@ small_tree = Dataset(meta_path / "small_tree.yaml") toy_graph = Dataset(meta_path / "toy_graph.yaml") toy_graph_undirected = Dataset(meta_path / "toy_graph_undirected.yaml") + +# Benchmarking datasets: be mindful of memory usage +# 250 MB +soc_livejournal = Dataset(meta_path / "soc-livejournal1.yaml") +# 965 MB +cit_patents = Dataset(meta_path / "cit-patents.yaml") +# 1.8 GB +europe_osm = Dataset(meta_path / "europe_osm.yaml") +# 1.5 GB +hollywood = Dataset(meta_path / "hollywood.yaml") diff --git a/python/cugraph/cugraph/datasets/dataset.py b/python/cugraph/cugraph/datasets/dataset.py index 877eade7708..dd7aa0df00a 100644 --- a/python/cugraph/cugraph/datasets/dataset.py +++ b/python/cugraph/cugraph/datasets/dataset.py @@ -14,44 +14,45 @@ import cudf import yaml import os +import pandas as pd from pathlib import Path from cugraph.structure.graph_classes import Graph class DefaultDownloadDir: """ - Maintains the path to the download directory used by Dataset instances. + Maintains a path to be used as a default download directory. + + All DefaultDownloadDir instances are based on RAPIDS_DATASET_ROOT_DIR if + set, or _default_base_dir if not set. + Instances of this class are typically shared by several Dataset instances in order to allow for the download directory to be defined and updated by a single object. """ - def __init__(self): - self._path = Path( - os.environ.get("RAPIDS_DATASET_ROOT_DIR", Path.home() / ".cugraph/datasets") - ) + _default_base_dir = Path.home() / ".cugraph/datasets" - @property - def path(self): + def __init__(self, *, subdir=""): """ - If `path` is not set, set it to the environment variable - RAPIDS_DATASET_ROOT_DIR. If the variable is not set, default to the - user's home directory. + subdir can be specified to provide a specialized dir under the base dir. """ - if self._path is None: - self._path = Path( - os.environ.get( - "RAPIDS_DATASET_ROOT_DIR", Path.home() / ".cugraph/datasets" - ) - ) - return self._path + self._subdir = Path(subdir) + self.reset() + + @property + def path(self): + return self._path.absolute() @path.setter def path(self, new): self._path = Path(new) - def clear(self): - self._path = None + def reset(self): + self._basedir = Path( + os.environ.get("RAPIDS_DATASET_ROOT_DIR", self._default_base_dir) + ) + self._path = self._basedir / self._subdir default_download_dir = DefaultDownloadDir() @@ -159,7 +160,7 @@ def unload(self): """ self._edgelist = None - def get_edgelist(self, download=False): + def get_edgelist(self, download=False, reader="cudf"): """ Return an Edgelist @@ -168,6 +169,9 @@ def get_edgelist(self, download=False): download : Boolean (default=False) Automatically download the dataset from the 'url' location within the YAML file. + + reader : 'cudf' or 'pandas' (default='cudf') + The library used to read a CSV and return an edgelist DataFrame. """ if self._edgelist is None: full_path = self.get_path() @@ -180,14 +184,29 @@ def get_edgelist(self, download=False): " exist. Try setting download=True" " to download the datafile" ) + header = None if isinstance(self.metadata["header"], int): header = self.metadata["header"] - self._edgelist = cudf.read_csv( - full_path, + + if reader == "cudf": + self.__reader = cudf.read_csv + elif reader == "pandas": + self.__reader = pd.read_csv + else: + raise ValueError( + "reader must be a module with a read_csv function compatible with \ + cudf.read_csv" + ) + + self._edgelist = self.__reader( + filepath_or_buffer=full_path, delimiter=self.metadata["delim"], names=self.metadata["col_names"], - dtype=self.metadata["col_types"], + dtype={ + self.metadata["col_names"][i]: self.metadata["col_types"][i] + for i in range(len(self.metadata["col_types"])) + }, header=header, ) @@ -219,6 +238,10 @@ def get_graph( dataset -if present- will be applied to the Graph. If the dataset does not contain weights, the Graph returned will be unweighted regardless of ignore_weights. + + store_transposed: Boolean (default=False) + If True, stores the transpose of the adjacency matrix. Required + for certain algorithms, such as pagerank. """ if self._edgelist is None: self.get_edgelist(download) @@ -237,20 +260,19 @@ def get_graph( "(or subclass) type or instance, got: " f"{type(create_using)}" ) - if len(self.metadata["col_names"]) > 2 and not (ignore_weights): G.from_cudf_edgelist( self._edgelist, - source="src", - destination="dst", - edge_attr="wgt", + source=self.metadata["col_names"][0], + destination=self.metadata["col_names"][1], + edge_attr=self.metadata["col_names"][2], store_transposed=store_transposed, ) else: G.from_cudf_edgelist( self._edgelist, - source="src", - destination="dst", + source=self.metadata["col_names"][0], + destination=self.metadata["col_names"][1], store_transposed=store_transposed, ) return G @@ -331,7 +353,7 @@ def download_all(force=False): def set_download_dir(path): """ - Set the download location fors datasets + Set the download location for datasets Parameters ---------- @@ -339,10 +361,10 @@ def set_download_dir(path): Location used to store datafiles """ if path is None: - default_download_dir.clear() + default_download_dir.reset() else: default_download_dir.path = path def get_download_dir(): - return default_download_dir.path.absolute() + return default_download_dir.path diff --git a/python/cugraph/cugraph/datasets/metadata/cit-patents.yaml b/python/cugraph/cugraph/datasets/metadata/cit-patents.yaml new file mode 100644 index 00000000000..d5c4cf195bd --- /dev/null +++ b/python/cugraph/cugraph/datasets/metadata/cit-patents.yaml @@ -0,0 +1,22 @@ +name: cit-Patents +file_type: .csv +description: A citation graph that includes all citations made by patents granted between 1975 and 1999, totaling 16,522,438 citations. +author: NBER +refs: + J. Leskovec, J. Kleinberg and C. Faloutsos. Graphs over Time Densification Laws, Shrinking Diameters and Possible Explanations. + ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD), 2005. +delim: " " +header: None +col_names: + - src + - dst +col_types: + - int32 + - int32 +has_loop: true +is_directed: true +is_multigraph: false +is_symmetric: false +number_of_edges: 16518948 +number_of_nodes: 3774768 +url: https://data.rapids.ai/cugraph/datasets/cit-Patents.csv \ No newline at end of file diff --git a/python/cugraph/cugraph/datasets/metadata/europe_osm.yaml b/python/cugraph/cugraph/datasets/metadata/europe_osm.yaml new file mode 100644 index 00000000000..fe0e42a4b86 --- /dev/null +++ b/python/cugraph/cugraph/datasets/metadata/europe_osm.yaml @@ -0,0 +1,21 @@ +name: europe_osm +file_type: .csv +description: A graph of OpenStreetMap data for Europe. +author: M. Kobitzsh / Geofabrik GmbH +refs: + Rossi, Ryan. Ahmed, Nesreen. The Network Data Respoistory with Interactive Graph Analytics and Visualization. +delim: " " +header: None +col_names: + - src + - dst +col_types: + - int32 + - int32 +has_loop: false +is_directed: false +is_multigraph: false +is_symmetric: true +number_of_edges: 54054660 +number_of_nodes: 50912018 +url: https://data.rapids.ai/cugraph/datasets/europe_osm.csv \ No newline at end of file diff --git a/python/cugraph/cugraph/datasets/metadata/hollywood.yaml b/python/cugraph/cugraph/datasets/metadata/hollywood.yaml new file mode 100644 index 00000000000..2f09cf7679b --- /dev/null +++ b/python/cugraph/cugraph/datasets/metadata/hollywood.yaml @@ -0,0 +1,26 @@ +name: hollywood +file_type: .csv +description: + A graph of movie actors where vertices are actors, and two actors are + joined by an edge whenever they appeared in a movie together. +author: Laboratory for Web Algorithmics (LAW) +refs: + The WebGraph Framework I Compression Techniques, Paolo Boldi + and Sebastiano Vigna, Proc. of the Thirteenth International + World Wide Web Conference (WWW 2004), 2004, Manhattan, USA, + pp. 595--601, ACM Press. +delim: " " +header: None +col_names: + - src + - dst +col_types: + - int32 + - int32 +has_loop: false +is_directed: false +is_multigraph: false +is_symmetric: true +number_of_edges: 57515616 +number_of_nodes: 1139905 +url: https://data.rapids.ai/cugraph/datasets/hollywood.csv \ No newline at end of file diff --git a/python/cugraph/cugraph/datasets/metadata/soc-livejournal1.yaml b/python/cugraph/cugraph/datasets/metadata/soc-livejournal1.yaml new file mode 100644 index 00000000000..fafc68acb9b --- /dev/null +++ b/python/cugraph/cugraph/datasets/metadata/soc-livejournal1.yaml @@ -0,0 +1,22 @@ +name: soc-LiveJournal1 +file_type: .csv +description: A graph of the LiveJournal social network. +author: L. Backstrom, D. Huttenlocher, J. Kleinberg, X. Lan +refs: + L. Backstrom, D. Huttenlocher, J. Kleinberg, X. Lan. Group Formation in + Large Social Networks Membership, Growth, and Evolution. KDD, 2006. +delim: " " +header: None +col_names: + - src + - dst +col_types: + - int32 + - int32 +has_loop: true +is_directed: true +is_multigraph: false +is_symmetric: false +number_of_edges: 68993773 +number_of_nodes: 4847571 +url: https://data.rapids.ai/cugraph/datasets/soc-LiveJournal1.csv \ No newline at end of file diff --git a/python/cugraph/cugraph/datasets/metadata/soc-twitter-2010.yaml b/python/cugraph/cugraph/datasets/metadata/soc-twitter-2010.yaml new file mode 100644 index 00000000000..df5df5735af --- /dev/null +++ b/python/cugraph/cugraph/datasets/metadata/soc-twitter-2010.yaml @@ -0,0 +1,22 @@ +name: soc-twitter-2010 +file_type: .csv +description: A network of follower relationships from a snapshot of Twitter in 2010, where an edge from i to j indicates that j is a follower of i. +author: H. Kwak, C. Lee, H. Park, S. Moon +refs: + J. Yang, J. Leskovec. Temporal Variation in Online Media. ACM Intl. + Conf. on Web Search and Data Mining (WSDM '11), 2011. +delim: " " +header: None +col_names: + - src + - dst +col_types: + - int32 + - int32 +has_loop: false +is_directed: false +is_multigraph: false +is_symmetric: false +number_of_edges: 530051354 +number_of_nodes: 21297772 +url: https://data.rapids.ai/cugraph/datasets/soc-twitter-2010.csv \ No newline at end of file diff --git a/python/cugraph/cugraph/experimental/datasets/__init__.py b/python/cugraph/cugraph/experimental/datasets/__init__.py deleted file mode 100644 index 18220243df1..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from cugraph.experimental.datasets.dataset import ( - Dataset, - load_all, - set_download_dir, - get_download_dir, - default_download_dir, -) -from cugraph.experimental.datasets import metadata -from pathlib import Path - -from cugraph.utilities.api_tools import promoted_experimental_warning_wrapper - - -Dataset = promoted_experimental_warning_wrapper(Dataset) -load_all = promoted_experimental_warning_wrapper(load_all) -set_download_dir = promoted_experimental_warning_wrapper(set_download_dir) -get_download_dir = promoted_experimental_warning_wrapper(get_download_dir) - -meta_path = Path(__file__).parent / "metadata" - - -# individual dataset objects -karate = Dataset(meta_path / "karate.yaml") -karate_data = Dataset(meta_path / "karate_data.yaml") -karate_undirected = Dataset(meta_path / "karate_undirected.yaml") -karate_asymmetric = Dataset(meta_path / "karate_asymmetric.yaml") -karate_disjoint = Dataset(meta_path / "karate-disjoint.yaml") -dolphins = Dataset(meta_path / "dolphins.yaml") -polbooks = Dataset(meta_path / "polbooks.yaml") -netscience = Dataset(meta_path / "netscience.yaml") -cyber = Dataset(meta_path / "cyber.yaml") -small_line = Dataset(meta_path / "small_line.yaml") -small_tree = Dataset(meta_path / "small_tree.yaml") -toy_graph = Dataset(meta_path / "toy_graph.yaml") -toy_graph_undirected = Dataset(meta_path / "toy_graph_undirected.yaml") -email_Eu_core = Dataset(meta_path / "email-Eu-core.yaml") -ktruss_polbooks = Dataset(meta_path / "ktruss_polbooks.yaml") - - -# batches of datasets -DATASETS_UNDIRECTED = [karate, dolphins] - -DATASETS_UNDIRECTED_WEIGHTS = [netscience] - -DATASETS_UNRENUMBERED = [karate_disjoint] - -DATASETS = [dolphins, netscience, karate_disjoint] - -DATASETS_SMALL = [karate, dolphins, polbooks] - -STRONGDATASETS = [dolphins, netscience, email_Eu_core] - -DATASETS_KTRUSS = [(polbooks, ktruss_polbooks)] - -MEDIUM_DATASETS = [polbooks] - -SMALL_DATASETS = [karate, dolphins, netscience] - -RLY_SMALL_DATASETS = [small_line, small_tree] - -ALL_DATASETS = [karate, dolphins, netscience, polbooks, small_line, small_tree] - -ALL_DATASETS_WGT = [karate, dolphins, netscience, polbooks, small_line, small_tree] - -TEST_GROUP = [dolphins, netscience] diff --git a/python/cugraph/cugraph/experimental/datasets/dataset.py b/python/cugraph/cugraph/experimental/datasets/dataset.py deleted file mode 100644 index 6b395d50fef..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/dataset.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import cudf -import yaml -import os -from pathlib import Path -from cugraph.structure.graph_classes import Graph - - -class DefaultDownloadDir: - """ - Maintains the path to the download directory used by Dataset instances. - Instances of this class are typically shared by several Dataset instances - in order to allow for the download directory to be defined and updated by - a single object. - """ - - def __init__(self): - self._path = Path( - os.environ.get("RAPIDS_DATASET_ROOT_DIR", Path.home() / ".cugraph/datasets") - ) - - @property - def path(self): - """ - If `path` is not set, set it to the environment variable - RAPIDS_DATASET_ROOT_DIR. If the variable is not set, default to the - user's home directory. - """ - if self._path is None: - self._path = Path( - os.environ.get( - "RAPIDS_DATASET_ROOT_DIR", Path.home() / ".cugraph/datasets" - ) - ) - return self._path - - @path.setter - def path(self, new): - self._path = Path(new) - - def clear(self): - self._path = None - - -default_download_dir = DefaultDownloadDir() - - -class Dataset: - """ - A Dataset Object, used to easily import edgelist data and cuGraph.Graph - instances. - - Parameters - ---------- - meta_data_file_name : yaml file - The metadata file for the specific graph dataset, which includes - information on the name, type, url link, data loading format, graph - properties - """ - - def __init__( - self, - metadata_yaml_file=None, - csv_file=None, - csv_header=None, - csv_delim=" ", - csv_col_names=None, - csv_col_dtypes=None, - ): - self._metadata_file = None - self._dl_path = default_download_dir - self._edgelist = None - self._path = None - - if metadata_yaml_file is not None and csv_file is not None: - raise ValueError("cannot specify both metadata_yaml_file and csv_file") - - elif metadata_yaml_file is not None: - with open(metadata_yaml_file, "r") as file: - self.metadata = yaml.safe_load(file) - self._metadata_file = Path(metadata_yaml_file) - - elif csv_file is not None: - if csv_col_names is None or csv_col_dtypes is None: - raise ValueError( - "csv_col_names and csv_col_dtypes must both be " - "not None when csv_file is specified." - ) - self._path = Path(csv_file) - if self._path.exists() is False: - raise FileNotFoundError(csv_file) - self.metadata = { - "name": self._path.with_suffix("").name, - "file_type": ".csv", - "url": None, - "header": csv_header, - "delim": csv_delim, - "col_names": csv_col_names, - "col_types": csv_col_dtypes, - } - - else: - raise ValueError("must specify either metadata_yaml_file or csv_file") - - def __str__(self): - """ - Use the basename of the meta_data_file the instance was constructed with, - without any extension, as the string repr. - """ - # The metadata file is likely to have a more descriptive file name, so - # use that one first if present. - # FIXME: this may need to provide a more unique or descriptive string repr - if self._metadata_file is not None: - return self._metadata_file.with_suffix("").name - else: - return self.get_path().with_suffix("").name - - def __download_csv(self, url): - """ - Downloads the .csv file from url to the current download path - (self._dl_path), updates self._path with the full path to the - downloaded file, and returns the latest value of self._path. - """ - self._dl_path.path.mkdir(parents=True, exist_ok=True) - - filename = self.metadata["name"] + self.metadata["file_type"] - if self._dl_path.path.is_dir(): - df = cudf.read_csv(url) - self._path = self._dl_path.path / filename - df.to_csv(self._path, index=False) - - else: - raise RuntimeError( - f"The directory {self._dl_path.path.absolute()}" "does not exist" - ) - return self._path - - def unload(self): - - """ - Remove all saved internal objects, forcing them to be re-created when - accessed. - - NOTE: This will cause calls to get_*() to re-read the dataset file from - disk. The caller should ensure the file on disk has not moved/been - deleted/changed. - """ - self._edgelist = None - - def get_edgelist(self, fetch=False): - """ - Return an Edgelist - - Parameters - ---------- - fetch : Boolean (default=False) - Automatically fetch for the dataset from the 'url' location within - the YAML file. - """ - if self._edgelist is None: - full_path = self.get_path() - if not full_path.is_file(): - if fetch: - full_path = self.__download_csv(self.metadata["url"]) - else: - raise RuntimeError( - f"The datafile {full_path} does not" - " exist. Try get_edgelist(fetch=True)" - " to download the datafile" - ) - header = None - if isinstance(self.metadata["header"], int): - header = self.metadata["header"] - self._edgelist = cudf.read_csv( - full_path, - delimiter=self.metadata["delim"], - names=self.metadata["col_names"], - dtype=self.metadata["col_types"], - header=header, - ) - - return self._edgelist - - def get_graph( - self, - fetch=False, - create_using=Graph, - ignore_weights=False, - store_transposed=False, - ): - """ - Return a Graph object. - - Parameters - ---------- - fetch : Boolean (default=False) - Downloads the dataset from the web. - - create_using: cugraph.Graph (instance or class), optional - (default=Graph) - Specify the type of Graph to create. Can pass in an instance to - create a Graph instance with specified 'directed' attribute. - - ignore_weights : Boolean (default=False) - Ignores weights in the dataset if True, resulting in an - unweighted Graph. If False (the default), weights from the - dataset -if present- will be applied to the Graph. If the - dataset does not contain weights, the Graph returned will - be unweighted regardless of ignore_weights. - """ - if self._edgelist is None: - self.get_edgelist(fetch) - - if create_using is None: - G = Graph() - elif isinstance(create_using, Graph): - # what about BFS if trnaposed is True - attrs = {"directed": create_using.is_directed()} - G = type(create_using)(**attrs) - elif type(create_using) is type: - G = create_using() - else: - raise TypeError( - "create_using must be a cugraph.Graph " - "(or subclass) type or instance, got: " - f"{type(create_using)}" - ) - - if len(self.metadata["col_names"]) > 2 and not (ignore_weights): - G.from_cudf_edgelist( - self._edgelist, - source="src", - destination="dst", - edge_attr="wgt", - store_transposed=store_transposed, - ) - else: - G.from_cudf_edgelist( - self._edgelist, - source="src", - destination="dst", - store_transposed=store_transposed, - ) - return G - - def get_path(self): - """ - Returns the location of the stored dataset file - """ - if self._path is None: - self._path = self._dl_path.path / ( - self.metadata["name"] + self.metadata["file_type"] - ) - - return self._path.absolute() - - -def load_all(force=False): - """ - Looks in `metadata` directory and fetches all datafiles from the the URLs - provided in each YAML file. - - Parameters - force : Boolean (default=False) - Overwrite any existing copies of datafiles. - """ - default_download_dir.path.mkdir(parents=True, exist_ok=True) - - meta_path = Path(__file__).parent.absolute() / "metadata" - for file in meta_path.iterdir(): - meta = None - if file.suffix == ".yaml": - with open(meta_path / file, "r") as metafile: - meta = yaml.safe_load(metafile) - - if "url" in meta: - filename = meta["name"] + meta["file_type"] - save_to = default_download_dir.path / filename - if not save_to.is_file() or force: - df = cudf.read_csv(meta["url"]) - df.to_csv(save_to, index=False) - - -def set_download_dir(path): - """ - Set the download directory for fetching datasets - - Parameters - ---------- - path : String - Location used to store datafiles - """ - if path is None: - default_download_dir.clear() - else: - default_download_dir.path = path - - -def get_download_dir(): - return default_download_dir.path.absolute() diff --git a/python/cugraph/cugraph/experimental/datasets/datasets_config.yaml b/python/cugraph/cugraph/experimental/datasets/datasets_config.yaml deleted file mode 100644 index 69a79db9cd9..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/datasets_config.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -fetch: "False" -force: "False" -# path where datasets will be downloaded to and stored -download_dir: "datasets" diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/cyber.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/cyber.yaml deleted file mode 100644 index 93ab5345442..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/cyber.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: cyber -file_type: .csv -author: N/A -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/cyber.csv -refs: N/A -col_names: - - idx - - srcip - - dstip -col_types: - - int32 - - str - - str -delim: "," -header: 0 -has_loop: true -is_directed: true -is_multigraph: false -is_symmetric: false -number_of_edges: 2546575 -number_of_nodes: 706529 -number_of_lines: 2546576 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/dolphins.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/dolphins.yaml deleted file mode 100644 index e4951375321..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/dolphins.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: dolphins -file_type: .csv -author: D. Lusseau -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/dolphins.csv -refs: - D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and S. M. Dawson, - The bottlenose dolphin community of Doubtful Sound features a large proportion of - long-lasting associations, Behavioral Ecology and Sociobiology 54, 396-405 (2003). -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -delim: " " -header: None -has_loop: false -is_directed: true -is_multigraph: false -is_symmetric: false -number_of_edges: 318 -number_of_nodes: 62 -number_of_lines: 318 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/email-Eu-core.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/email-Eu-core.yaml deleted file mode 100644 index 97d0dc82ee3..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/email-Eu-core.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: email-Eu-core -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/email-Eu-core.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: false -is_multigraph: false -is_symmetric: true -number_of_edges: 25571 -number_of_nodes: 1005 -number_of_lines: 25571 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/karate-disjoint.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/karate-disjoint.yaml deleted file mode 100644 index 0c0eaf78b63..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/karate-disjoint.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: karate-disjoint -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/karate-disjoint.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: True -is_multigraph: false -is_symmetric: true -number_of_edges: 312 -number_of_nodes: 68 -number_of_lines: 312 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/karate.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/karate.yaml deleted file mode 100644 index 273381ed368..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/karate.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: karate -file_type: .csv -author: Zachary W. -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/karate.csv -refs: - W. W. Zachary, An information flow model for conflict and fission in small groups, - Journal of Anthropological Research 33, 452-473 (1977). -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: true -is_directed: true -is_multigraph: false -is_symmetric: true -number_of_edges: 156 -number_of_nodes: 34 -number_of_lines: 156 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/karate_asymmetric.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/karate_asymmetric.yaml deleted file mode 100644 index 3616b8fb3a5..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/karate_asymmetric.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: karate-asymmetric -file_type: .csv -author: Zachary W. -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/karate-asymmetric.csv -delim: " " -header: None -refs: - W. W. Zachary, An information flow model for conflict and fission in small groups, - Journal of Anthropological Research 33, 452-473 (1977). -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: true -is_directed: false -is_multigraph: false -is_symmetric: false -number_of_edges: 78 -number_of_nodes: 34 -number_of_lines: 78 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/karate_data.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/karate_data.yaml deleted file mode 100644 index 9a8b27f21ae..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/karate_data.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: karate-data -file_type: .csv -author: Zachary W. -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/karate-data.csv -refs: - W. W. Zachary, An information flow model for conflict and fission in small groups, - Journal of Anthropological Research 33, 452-473 (1977). -delim: "\t" -header: None -col_names: - - src - - dst -col_types: - - int32 - - int32 -has_loop: true -is_directed: true -is_multigraph: false -is_symmetric: true -number_of_edges: 156 -number_of_nodes: 34 -number_of_lines: 156 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/karate_undirected.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/karate_undirected.yaml deleted file mode 100644 index 1b45f86caee..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/karate_undirected.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: karate_undirected -file_type: .csv -author: Zachary W. -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/karate_undirected.csv -refs: - W. W. Zachary, An information flow model for conflict and fission in small groups, - Journal of Anthropological Research 33, 452-473 (1977). -delim: "\t" -header: None -col_names: - - src - - dst -col_types: - - int32 - - int32 -has_loop: true -is_directed: false -is_multigraph: false -is_symmetric: true -number_of_edges: 78 -number_of_nodes: 34 -number_of_lines: 78 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/ktruss_polbooks.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/ktruss_polbooks.yaml deleted file mode 100644 index 1ef29b3917e..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/ktruss_polbooks.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: ktruss_polbooks -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/ref/ktruss/polbooks.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: true -is_multigraph: false -is_symmetric: false -number_of_edges: 233 -number_of_nodes: 58 -number_of_lines: 233 - diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/netscience.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/netscience.yaml deleted file mode 100644 index 2dca702df3d..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/netscience.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: netscience -file_type: .csv -author: Newman, Mark EJ -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/netscience.csv -refs: Finding community structure in networks using the eigenvectors of matrices. -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: true -is_multigraph: false -is_symmetric: true -number_of_edges: 2742 -number_of_nodes: 1461 -number_of_lines: 5484 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/polbooks.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/polbooks.yaml deleted file mode 100644 index 5816e5672fd..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/polbooks.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: polbooks -file_type: .csv -author: V. Krebs -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/polbooks.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -is_directed: true -has_loop: null -is_multigraph: null -is_symmetric: true -number_of_edges: 882 -number_of_nodes: 105 -number_of_lines: 882 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/small_line.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/small_line.yaml deleted file mode 100644 index 5b724ac99fd..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/small_line.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: small_line -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/small_line.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: false -is_multigraph: false -is_symmetric: true -number_of_edges: 9 -number_of_nodes: 10 -number_of_lines: 8 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/small_tree.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/small_tree.yaml deleted file mode 100644 index 8eeac346d2a..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/small_tree.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: small_tree -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/small_tree.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: true -is_multigraph: false -is_symmetric: true -number_of_edges: 11 -number_of_nodes: 9 -number_of_lines: 11 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/toy_graph.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/toy_graph.yaml deleted file mode 100644 index 819aad06f6a..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/toy_graph.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: toy_graph -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/toy_graph.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: false -is_multigraph: false -is_symmetric: true -number_of_edges: 16 -number_of_nodes: 6 -number_of_lines: 16 diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/toy_graph_undirected.yaml b/python/cugraph/cugraph/experimental/datasets/metadata/toy_graph_undirected.yaml deleted file mode 100644 index c6e86bdf334..00000000000 --- a/python/cugraph/cugraph/experimental/datasets/metadata/toy_graph_undirected.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: toy_graph_undirected -file_type: .csv -author: null -url: https://raw.githubusercontent.com/rapidsai/cugraph/branch-22.08/datasets/toy_graph_undirected.csv -refs: null -delim: " " -header: None -col_names: - - src - - dst - - wgt -col_types: - - int32 - - int32 - - float32 -has_loop: false -is_directed: false -is_multigraph: false -is_symmetric: true -number_of_edges: 8 -number_of_nodes: 6 -number_of_lines: 8 diff --git a/python/cugraph/cugraph/structure/__init__.py b/python/cugraph/cugraph/structure/__init__.py index d7e0ff62358..94f34fd23f3 100644 --- a/python/cugraph/cugraph/structure/__init__.py +++ b/python/cugraph/cugraph/structure/__init__.py @@ -25,6 +25,11 @@ ) from cugraph.structure.number_map import NumberMap from cugraph.structure.symmetrize import symmetrize, symmetrize_df, symmetrize_ddf +from cugraph.structure.replicate_edgelist import ( + replicate_edgelist, + replicate_cudf_dataframe, + replicate_cudf_series, +) from cugraph.structure.convert_matrix import ( from_edgelist, from_cudf_edgelist, diff --git a/python/cugraph/cugraph/structure/graph_classes.py b/python/cugraph/cugraph/structure/graph_classes.py index 6f6c7e5a26c..03efcba0307 100644 --- a/python/cugraph/cugraph/structure/graph_classes.py +++ b/python/cugraph/cugraph/structure/graph_classes.py @@ -469,8 +469,7 @@ def from_numpy_array(self, np_array, nodes=None): nodes: array-like or None, optional (default=None) A list of column names, acting as labels for nodes """ - if not isinstance(np_array, np.ndarray): - raise TypeError("np_array input is not a Numpy array") + np_array = np.asarray(np_array) if len(np_array.shape) != 2: raise ValueError("np_array is not a 2D matrix") diff --git a/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py b/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py index 935d0c597d4..f666900b226 100644 --- a/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py +++ b/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py @@ -36,8 +36,8 @@ from cugraph.structure.symmetrize import symmetrize from cugraph.dask.common.part_utils import ( get_persisted_df_worker_map, - get_length_of_parts, persist_dask_df_equal_parts_per_worker, + _chunk_lst, ) from cugraph.dask import get_n_workers import cugraph.dask.comms.comms as Comms @@ -81,6 +81,10 @@ def __init__(self, properties): self.destination_columns = None self.weight_column = None self.vertex_columns = None + self.vertex_type = None + self.weight_type = None + self.edge_id_type = None + self.edge_type_id_type = None def _make_plc_graph( sID, @@ -89,51 +93,69 @@ def _make_plc_graph( src_col_name, dst_col_name, store_transposed, - num_edges, + vertex_type, + weight_type, + edge_id_type, + edge_type_id, ): - weights = None edge_ids = None edge_types = None - if simpleDistributedGraphImpl.edgeWeightCol in edata_x[0]: - weights = _get_column_from_ls_dfs( - edata_x, simpleDistributedGraphImpl.edgeWeightCol - ) - if weights.dtype == "int32": - weights = weights.astype("float32") - elif weights.dtype == "int64": - weights = weights.astype("float64") - - if simpleDistributedGraphImpl.edgeIdCol in edata_x[0]: - edge_ids = _get_column_from_ls_dfs( - edata_x, simpleDistributedGraphImpl.edgeIdCol - ) - if edata_x[0][src_col_name].dtype == "int64" and edge_ids.dtype != "int64": - edge_ids = edge_ids.astype("int64") + num_arrays = len(edata_x) + if weight_type is not None: + weights = [ + edata_x[i][simpleDistributedGraphImpl.edgeWeightCol] + for i in range(num_arrays) + ] + if weight_type == "int32": + weights = [w_array.astype("float32") for w_array in weights] + elif weight_type == "int64": + weights = [w_array.astype("float64") for w_array in weights] + + if edge_id_type is not None: + edge_ids = [ + edata_x[i][simpleDistributedGraphImpl.edgeIdCol] + for i in range(num_arrays) + ] + if vertex_type == "int64" and edge_id_type != "int64": + edge_ids = [e_id_array.astype("int64") for e_id_array in edge_ids] warnings.warn( - f"Vertex type is int64 but edge id type is {edge_ids.dtype}" + f"Vertex type is int64 but edge id type is {edge_ids[0].dtype}" ", automatically casting edge id type to int64. " "This may cause extra memory usage. Consider passing" " a int64 list of edge ids instead." ) - if simpleDistributedGraphImpl.edgeTypeCol in edata_x[0]: - edge_types = _get_column_from_ls_dfs( - edata_x, simpleDistributedGraphImpl.edgeTypeCol - ) + if edge_type_id is not None: + edge_types = [ + edata_x[i][simpleDistributedGraphImpl.edgeTypeCol] + for i in range(num_arrays) + ] - return MGGraph( + src_array = [edata_x[i][src_col_name] for i in range(num_arrays)] + dst_array = [edata_x[i][dst_col_name] for i in range(num_arrays)] + plc_graph = MGGraph( resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), graph_properties=graph_props, - src_array=_get_column_from_ls_dfs(edata_x, src_col_name), - dst_array=_get_column_from_ls_dfs(edata_x, dst_col_name), - weight_array=weights, - edge_id_array=edge_ids, - edge_type_array=edge_types, + src_array=src_array if src_array else cudf.Series(dtype=vertex_type), + dst_array=dst_array if dst_array else cudf.Series(dtype=vertex_type), + weight_array=weights + if weights + else ([cudf.Series(dtype=weight_type)] if weight_type else None), + edge_id_array=edge_ids + if edge_ids + else ([cudf.Series(dtype=edge_id_type)] if edge_id_type else None), + edge_type_array=edge_types + if edge_types + else ([cudf.Series(dtype=edge_type_id)] if edge_type_id else None), + num_arrays=num_arrays, store_transposed=store_transposed, - num_edges=num_edges, do_expensive_check=False, ) + del edata_x + gc.collect() + + return plc_graph # Functions def __from_edgelist( @@ -182,7 +204,6 @@ def __from_edgelist( workers = _client.scheduler_info()["workers"] # Repartition to 2 partitions per GPU for memory efficient process input_ddf = input_ddf.repartition(npartitions=len(workers) * 2) - input_ddf = input_ddf.map_partitions(lambda df: df.copy()) # The dataframe will be symmetrized iff the graph is undirected # otherwise, the inital dataframe will be returned if edge_attr is not None: @@ -314,19 +335,25 @@ def __from_edgelist( dst_col_name = self.renumber_map.renumbered_dst_col_name ddf = self.edgelist.edgelist_df + + # Get the edgelist dtypes + self.vertex_type = ddf[src_col_name].dtype + if simpleDistributedGraphImpl.edgeWeightCol in ddf.columns: + self.weight_type = ddf[simpleDistributedGraphImpl.edgeWeightCol].dtype + if simpleDistributedGraphImpl.edgeIdCol in ddf.columns: + self.edge_id_type = ddf[simpleDistributedGraphImpl.edgeIdCol].dtype + if simpleDistributedGraphImpl.edgeTypeCol in ddf.columns: + self.edge_type_id_type = ddf[simpleDistributedGraphImpl.edgeTypeCol].dtype + graph_props = GraphProperties( is_multigraph=self.properties.multi_edge, is_symmetric=not self.properties.directed, ) ddf = ddf.repartition(npartitions=len(workers) * 2) - persisted_keys_d = persist_dask_df_equal_parts_per_worker( - ddf, _client, return_type="dict" - ) - del ddf - length_of_parts = get_length_of_parts(persisted_keys_d, _client) - num_edges = sum( - [item for sublist in length_of_parts.values() for item in sublist] - ) + ddf_keys = ddf.to_delayed() + workers = _client.scheduler_info()["workers"].keys() + ddf_keys_ls = _chunk_lst(ddf_keys, len(workers)) + delayed_tasks_d = { w: delayed(simpleDistributedGraphImpl._make_plc_graph)( Comms.get_session_id(), @@ -335,9 +362,12 @@ def __from_edgelist( src_col_name, dst_col_name, store_transposed, - num_edges, + self.vertex_type, + self.weight_type, + self.edge_id_type, + self.edge_type_id_type, ) - for w, edata in persisted_keys_d.items() + for w, edata in zip(workers, ddf_keys_ls) } self._plc_graph = { w: _client.compute( @@ -346,8 +376,9 @@ def __from_edgelist( for w, delayed_task in delayed_tasks_d.items() } wait(list(self._plc_graph.values())) - del persisted_keys_d + del ddf_keys del delayed_tasks_d + gc.collect() _client.run(gc.collect) @property @@ -1189,18 +1220,3 @@ def vertex_column_size(self): @property def _npartitions(self) -> int: return len(self._plc_graph) - - -def _get_column_from_ls_dfs(lst_df, col_name): - """ - This function concatenates the column - and drops it from the input list - """ - len_df = sum([len(df) for df in lst_df]) - if len_df == 0: - return lst_df[0][col_name] - output_col = cudf.concat([df[col_name] for df in lst_df], ignore_index=True) - for df in lst_df: - df.drop(columns=[col_name], inplace=True) - gc.collect() - return output_col diff --git a/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py b/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py index 2b23d3a26b7..22d82eb1796 100644 --- a/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py +++ b/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py @@ -1286,9 +1286,13 @@ def nodes(self): else: return df[df.columns[0]] else: - return cudf.concat( - [df[simpleGraphImpl.srcCol], df[simpleGraphImpl.dstCol]] - ).unique() + return ( + cudf.concat( + [df[simpleGraphImpl.srcCol], df[simpleGraphImpl.dstCol]] + ) + .drop_duplicates() + .reset_index(drop=True) + ) if self.adjlist is not None: return cudf.Series(np.arange(0, self.number_of_nodes())) diff --git a/python/cugraph/cugraph/structure/property_graph.py b/python/cugraph/cugraph/structure/property_graph.py index 36ce5baa212..513798f35f9 100644 --- a/python/cugraph/cugraph/structure/property_graph.py +++ b/python/cugraph/cugraph/structure/property_graph.py @@ -800,15 +800,9 @@ def add_vertex_data( tmp_df.index = tmp_df.index.rename(self.vertex_col_name) # FIXME: handle case of a type_name column already being in tmp_df - if self.__series_type is cudf.Series: - # cudf does not yet support initialization with a scalar - tmp_df[TCN] = cudf.Series( - cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)), - index=tmp_df.index, - ) - else: - # pandas is oddly slow if dtype is passed to the constructor here - tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype) + tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype( + cat_dtype + ) if property_columns: # all columns @@ -1207,15 +1201,9 @@ def add_edge_data( tmp_df[self.src_col_name] = tmp_df[vertex_col_names[0]] tmp_df[self.dst_col_name] = tmp_df[vertex_col_names[1]] - if self.__series_type is cudf.Series: - # cudf does not yet support initialization with a scalar - tmp_df[TCN] = cudf.Series( - cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)), - index=tmp_df.index, - ) - else: - # pandas is oddly slow if dtype is passed to the constructor here - tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype) + tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype( + cat_dtype + ) # Add unique edge IDs to the new rows. This is just a count for each # row starting from the last edge ID value, with initial edge ID 0. diff --git a/python/cugraph/cugraph/structure/replicate_edgelist.py b/python/cugraph/cugraph/structure/replicate_edgelist.py new file mode 100644 index 00000000000..d413e50e485 --- /dev/null +++ b/python/cugraph/cugraph/structure/replicate_edgelist.py @@ -0,0 +1,351 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dask_cudf +import cudf +from dask.distributed import wait, default_client +import numpy as np +from pylibcugraph import ( + ResourceHandle, + replicate_edgelist as pylibcugraph_replicate_edgelist, +) + +from cugraph.dask.common.part_utils import ( + get_persisted_df_worker_map, + persist_dask_df_equal_parts_per_worker, +) + +import dask +import cupy as cp +import cugraph.dask.comms.comms as Comms +from typing import Union, Tuple + + +# FIXME: Convert it to a general-purpose util function +def _convert_to_cudf(cp_arrays: Tuple[cp.ndarray], col_names: list) -> cudf.DataFrame: + """ + Creates a cudf Dataframe from cupy arrays + """ + src, dst, wgt, edge_id, edge_type_id, _ = cp_arrays + gathered_edgelist_df = cudf.DataFrame() + gathered_edgelist_df[col_names[0]] = src + gathered_edgelist_df[col_names[1]] = dst + if wgt is not None: + gathered_edgelist_df[col_names[2]] = wgt + if edge_id is not None: + gathered_edgelist_df[col_names[3]] = edge_id + if edge_type_id is not None: + gathered_edgelist_df[col_names[4]] = edge_type_id + + return gathered_edgelist_df + + +def _call_plc_replicate_edgelist( + sID: bytes, edgelist_df: cudf.DataFrame, col_names: list +) -> cudf.DataFrame: + edgelist_df = edgelist_df[0] + cp_arrays = pylibcugraph_replicate_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + src_array=edgelist_df[col_names[0]], + dst_array=edgelist_df[col_names[1]], + weight_array=edgelist_df[col_names[2]] if len(col_names) > 2 else None, + edge_id_array=edgelist_df[col_names[3]] if len(col_names) > 3 else None, + edge_type_id_array=edgelist_df[col_names[4]] if len(col_names) > 4 else None, + ) + return _convert_to_cudf(cp_arrays, col_names) + + +def _call_plc_replicate_dataframe(sID: bytes, df: cudf.DataFrame) -> cudf.DataFrame: + df = df[0] + df_replicated = cudf.DataFrame() + for col_name in df.columns: + cp_array = pylibcugraph_replicate_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + src_array=df[col_name] + if df[col_name].dtype in [np.int32, np.int64] + else None, + dst_array=None, + weight_array=df[col_name] + if df[col_name].dtype in [np.float32, np.float64] + else None, + edge_id_array=None, + edge_type_id_array=None, + ) + src, _, wgt, _, _, _ = cp_array + if src is not None: + df_replicated[col_name] = src + elif wgt is not None: + df_replicated[col_name] = wgt + + return df_replicated + + +def _call_plc_replicate_series(sID: bytes, series: cudf.Series) -> cudf.Series: + series = series[0] + series_replicated = cudf.Series() + cp_array = pylibcugraph_replicate_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + src_array=series if series.dtype in [np.int32, np.int64] else None, + dst_array=None, + weight_array=series if series.dtype in [np.float32, np.float64] else None, + edge_id_array=None, + edge_type_id_array=None, + ) + src, _, wgt, _, _, _ = cp_array + if src is not None: + series_replicated = cudf.Series(src) + elif wgt is not None: + series_replicated = cudf.Series(wgt) + + return series_replicated + + +def _mg_call_plc_replicate( + client: dask.distributed.client.Client, + sID: bytes, + dask_object: dict, + input_type: str, + col_names: list, +) -> Union[dask_cudf.DataFrame, dask_cudf.Series]: + + if input_type == "dataframe": + result = [ + client.submit( + _call_plc_replicate_dataframe, + sID, + edata, + workers=[w], + allow_other_workers=False, + pure=False, + ) + for w, edata in dask_object.items() + ] + elif input_type == "dataframe": + result = [ + client.submit( + _call_plc_replicate_series, + sID, + edata, + workers=[w], + allow_other_workers=False, + pure=False, + ) + for w, edata in dask_object.items() + ] + elif input_type == "edgelist": + result = [ + client.submit( + _call_plc_replicate_edgelist, + sID, + edata, + col_names, + workers=[w], + allow_other_workers=False, + pure=False, + ) + for w, edata in dask_object.items() + ] + + ddf = dask_cudf.from_delayed(result, verify_meta=False).persist() + wait(ddf) + wait([r.release() for r in result]) + return ddf + + +def replicate_edgelist( + edgelist_ddf: Union[dask_cudf.DataFrame, cudf.DataFrame] = None, + source="src", + destination="dst", + weight=None, + edge_id=None, + edge_type=None, +) -> dask_cudf.DataFrame: + """ + Replicate edges across all GPUs + + Parameters + ---------- + + edgelist_ddf: cudf.DataFrame or dask_cudf.DataFrame + A DataFrame that contains edge information. + + source : str or array-like + source column name or array of column names + + destination : str or array-like + destination column name or array of column names + + weight : str, optional (default=None) + Name of the weight column in the input dataframe. + + edge_id : str, optional (default=None) + Name of the edge id column in the input dataframe. + + edge_type : str, optional (default=None) + Name of the edge type column in the input dataframe. + + Returns + ------- + df : dask_cudf.DataFrame + A distributed dataframe where each partition contains the + combined edgelist from all GPUs. If a cudf.DataFrame was passed + as input, the edgelist will be replicated across all the other + GPUs in the cluster. If as dask_cudf.DataFrame was passed as input, + each partition will be filled with the edges of all partitions + in the dask_cudf.DataFrame. + + """ + + _client = default_client() + + if isinstance(edgelist_ddf, cudf.DataFrame): + edgelist_ddf = dask_cudf.from_cudf( + edgelist_ddf, npartitions=len(Comms.get_workers()) + ) + col_names = [source, destination] + + if weight is not None: + col_names.append(weight) + if edge_id is not None: + col_names.append(edge_id) + if edge_type is not None: + col_names.append(edge_type) + + if not (set(col_names).issubset(set(edgelist_ddf.columns))): + raise ValueError( + "Invalid column names were provided: valid columns names are " + f"{edgelist_ddf.columns}" + ) + + edgelist_ddf = persist_dask_df_equal_parts_per_worker(edgelist_ddf, _client) + edgelist_ddf = get_persisted_df_worker_map(edgelist_ddf, _client) + + ddf = _mg_call_plc_replicate( + _client, + Comms.get_session_id(), + edgelist_ddf, + "edgelist", + col_names, + ) + + return ddf + + +def replicate_cudf_dataframe(cudf_dataframe): + """ + Replicate dataframe across all GPUs + + Parameters + ---------- + + cudf_dataframe: cudf.DataFrame or dask_cudf.DataFrame + + Returns + ------- + df : dask_cudf.DataFrame + A distributed dataframe where each partition contains the + combined dataframe from all GPUs. If a cudf.DataFrame was passed + as input, the dataframe will be replicated across all the other + GPUs in the cluster. If as dask_cudf.DataFrame was passed as input, + each partition will be filled with the datafame of all partitions + in the dask_cudf.DataFrame. + + """ + + supported_types = [np.int32, np.int64, np.float32, np.float64] + if not all(dtype in supported_types for dtype in cudf_dataframe.dtypes): + raise TypeError( + "The supported types are 'int32', 'int64', 'float32', 'float64'" + ) + + _client = default_client() + + if not isinstance(cudf_dataframe, dask_cudf.DataFrame): + if isinstance(cudf_dataframe, cudf.DataFrame): + df = dask_cudf.from_cudf( + cudf_dataframe, npartitions=len(Comms.get_workers()) + ) + elif not isinstance(cudf_dataframe, dask_cudf.DataFrame): + raise TypeError( + "The variable 'cudf_dataframe' must be of type " + f"'cudf/dask_cudf.dataframe', got type {type(cudf_dataframe)}" + ) + else: + df = cudf_dataframe + + df = persist_dask_df_equal_parts_per_worker(df, _client) + df = get_persisted_df_worker_map(df, _client) + + ddf = _mg_call_plc_replicate( + _client, + Comms.get_session_id(), + df, + "dataframe", + ) + + return ddf + + +def replicate_cudf_series(cudf_series): + """ + Replicate series across all GPUs + + Parameters + ---------- + + cudf_series: cudf.Series or dask_cudf.Series + + Returns + ------- + series : dask_cudf.Series + A distributed series where each partition contains the + combined series from all GPUs. If a cudf.Series was passed + as input, the Series will be replicated across all the other + GPUs in the cluster. If as dask_cudf.Series was passed as input, + each partition will be filled with the series of all partitions + in the dask_cudf.Series. + + """ + + supported_types = [np.int32, np.int64, np.float32, np.float64] + if cudf_series.dtype not in supported_types: + raise TypeError( + "The supported types are 'int32', 'int64', 'float32', 'float64'" + ) + + _client = default_client() + + if not isinstance(cudf_series, dask_cudf.Series): + if isinstance(cudf_series, cudf.Series): + series = dask_cudf.from_cudf( + cudf_series, npartitions=len(Comms.get_workers()) + ) + elif not isinstance(cudf_series, dask_cudf.Series): + raise TypeError( + "The variable 'cudf_series' must be of type " + f"'cudf/dask_cudf.series', got type {type(cudf_series)}" + ) + else: + series = cudf_series + + series = persist_dask_df_equal_parts_per_worker(series, _client) + series = get_persisted_df_worker_map(series, _client) + + series = _mg_call_plc_replicate( + _client, + Comms.get_session_id(), + series, + "series", + ) + + return series diff --git a/python/cugraph/cugraph/structure/symmetrize.py b/python/cugraph/cugraph/structure/symmetrize.py index 4c00e68344d..b324ff65834 100644 --- a/python/cugraph/cugraph/structure/symmetrize.py +++ b/python/cugraph/cugraph/structure/symmetrize.py @@ -299,10 +299,8 @@ def _memory_efficient_drop_duplicates(ddf, vertex_col_name, num_workers): Drop duplicate edges from the input dataframe. """ # drop duplicates has a 5x+ overhead - # and does not seem to be working as expected - # TODO: Triage an MRE ddf = ddf.reset_index(drop=True).repartition(npartitions=num_workers * 2) - ddf = ddf.groupby(by=[*vertex_col_name], as_index=False).min( - split_out=num_workers * 2 + ddf = ddf.drop_duplicates( + subset=[*vertex_col_name], ignore_index=True, split_out=num_workers * 2 ) return ddf diff --git a/python/cugraph/cugraph/testing/__init__.py b/python/cugraph/cugraph/testing/__init__.py index f5f0bcb06eb..2b4a4fd3ebf 100644 --- a/python/cugraph/cugraph/testing/__init__.py +++ b/python/cugraph/cugraph/testing/__init__.py @@ -19,7 +19,7 @@ Resultset, load_resultset, get_resultset, - results_dir_path, + default_resultset_download_dir, ) from cugraph.datasets import ( cyber, @@ -34,6 +34,11 @@ email_Eu_core, toy_graph, toy_graph_undirected, + soc_livejournal, + cit_patents, + europe_osm, + hollywood, + # twitter, ) # @@ -66,3 +71,4 @@ toy_graph_undirected, ] DEFAULT_DATASETS = [dolphins, netscience, karate_disjoint] +BENCHMARKING_DATASETS = [soc_livejournal, cit_patents, europe_osm, hollywood] diff --git a/python/cugraph/cugraph/testing/generate_resultsets.py b/python/cugraph/cugraph/testing/generate_resultsets.py index 9724aca32dc..2ae0f52d88b 100644 --- a/python/cugraph/cugraph/testing/generate_resultsets.py +++ b/python/cugraph/cugraph/testing/generate_resultsets.py @@ -20,8 +20,14 @@ import cudf import cugraph from cugraph.datasets import dolphins, netscience, karate_disjoint, karate -from cugraph.testing import utils, Resultset, SMALL_DATASETS, results_dir_path +# from cugraph.testing import utils, Resultset, SMALL_DATASETS, results_dir_path +from cugraph.testing import ( + utils, + Resultset, + SMALL_DATASETS, + default_resultset_download_dir, +) _resultsets = {} @@ -224,6 +230,7 @@ def add_resultset(result_data_dictionary, **kwargs): ] ) # Generating ALL results files + results_dir_path = default_resultset_download_dir.path if not results_dir_path.exists(): results_dir_path.mkdir(parents=True, exist_ok=True) diff --git a/python/cugraph/cugraph/testing/mg_utils.py b/python/cugraph/cugraph/testing/mg_utils.py index bd165ba3db5..32854652f05 100644 --- a/python/cugraph/cugraph/testing/mg_utils.py +++ b/python/cugraph/cugraph/testing/mg_utils.py @@ -33,6 +33,7 @@ def start_dask_client( rmm_pool_size=None, dask_worker_devices=None, jit_unspill=False, + worker_class=None, device_memory_limit=0.8, ): """ @@ -141,6 +142,7 @@ def start_dask_client( rmm_async=rmm_async, CUDA_VISIBLE_DEVICES=dask_worker_devices, jit_unspill=jit_unspill, + worker_class=worker_class, device_memory_limit=device_memory_limit, ) client = Client(cluster) diff --git a/python/cugraph/cugraph/testing/resultset.py b/python/cugraph/cugraph/testing/resultset.py index 490e3a7c4ff..9570d7f3e04 100644 --- a/python/cugraph/cugraph/testing/resultset.py +++ b/python/cugraph/cugraph/testing/resultset.py @@ -16,10 +16,12 @@ import urllib.request import cudf -from cugraph.testing import utils +from cugraph.datasets.dataset import ( + DefaultDownloadDir, + default_download_dir, +) - -results_dir_path = utils.RAPIDS_DATASET_ROOT_DIR_PATH / "tests" / "resultsets" +# results_dir_path = utils.RAPIDS_DATASET_ROOT_DIR_PATH / "tests" / "resultsets" class Resultset: @@ -48,6 +50,42 @@ def get_cudf_dataframe(self): _resultsets = {} +def get_resultset(resultset_name, **kwargs): + """ + Returns the golden results for a specific test. + + Parameters + ---------- + resultset_name : String + Name of the test's module (currently just 'traversal' is supported) + + kwargs : + All distinct test details regarding the choice of algorithm, dataset, + and graph + """ + arg_dict = dict(kwargs) + arg_dict["resultset_name"] = resultset_name + # Example: + # {'a': 1, 'z': 9, 'c': 5, 'b': 2} becomes 'a-1-b-2-c-5-z-9' + resultset_key = "-".join( + [ + str(val) + for arg_dict_pair in sorted(arg_dict.items()) + for val in arg_dict_pair + ] + ) + uuid = _resultsets.get(resultset_key) + if uuid is None: + raise KeyError(f"results for {arg_dict} not found") + + results_dir_path = default_resultset_download_dir.path + results_filename = results_dir_path / (uuid + ".csv") + return cudf.read_csv(results_filename) + + +default_resultset_download_dir = DefaultDownloadDir(subdir="tests/resultsets") + + def load_resultset(resultset_name, resultset_download_url): """ Read a mapping file (.csv) in the _results_dir and save the @@ -56,17 +94,21 @@ def load_resultset(resultset_name, resultset_download_url): _results_dir, use resultset_download_url to download a file to install/unpack/etc. to _results_dir first. """ - mapping_file_path = results_dir_path / (resultset_name + "_mappings.csv") + # curr_resultset_download_dir = get_resultset_download_dir() + curr_resultset_download_dir = default_resultset_download_dir.path + # curr_download_dir = path + curr_download_dir = default_download_dir.path + mapping_file_path = curr_resultset_download_dir / (resultset_name + "_mappings.csv") if not mapping_file_path.exists(): # Downloads a tar gz from s3 bucket, then unpacks the results files - compressed_file_dir = utils.RAPIDS_DATASET_ROOT_DIR_PATH / "tests" + compressed_file_dir = curr_download_dir / "tests" compressed_file_path = compressed_file_dir / "resultsets.tar.gz" - if not results_dir_path.exists(): - results_dir_path.mkdir(parents=True, exist_ok=True) + if not curr_resultset_download_dir.exists(): + curr_resultset_download_dir.mkdir(parents=True, exist_ok=True) if not compressed_file_path.exists(): urllib.request.urlretrieve(resultset_download_url, compressed_file_path) tar = tarfile.open(str(compressed_file_path), "r:gz") - tar.extractall(str(results_dir_path)) + tar.extractall(str(curr_resultset_download_dir)) tar.close() # FIXME: This assumes separator is " ", but should this be configurable? @@ -102,35 +144,3 @@ def load_resultset(resultset_name, resultset_download_url): ) _resultsets[resultset_key] = uuid - - -def get_resultset(resultset_name, **kwargs): - """ - Returns the golden results for a specific test. - - Parameters - ---------- - resultset_name : String - Name of the test's module (currently just 'traversal' is supported) - - kwargs : - All distinct test details regarding the choice of algorithm, dataset, - and graph - """ - arg_dict = dict(kwargs) - arg_dict["resultset_name"] = resultset_name - # Example: - # {'a': 1, 'z': 9, 'c': 5, 'b': 2} becomes 'a-1-b-2-c-5-z-9' - resultset_key = "-".join( - [ - str(val) - for arg_dict_pair in sorted(arg_dict.items()) - for val in arg_dict_pair - ] - ) - uuid = _resultsets.get(resultset_key) - if uuid is None: - raise KeyError(f"results for {arg_dict} not found") - - results_filename = results_dir_path / (uuid + ".csv") - return cudf.read_csv(results_filename) diff --git a/python/cugraph/cugraph/tests/centrality/test_edge_betweenness_centrality_mg.py b/python/cugraph/cugraph/tests/centrality/test_edge_betweenness_centrality_mg.py index 4277f94a396..478b7e655d5 100644 --- a/python/cugraph/cugraph/tests/centrality/test_edge_betweenness_centrality_mg.py +++ b/python/cugraph/cugraph/tests/centrality/test_edge_betweenness_centrality_mg.py @@ -16,7 +16,7 @@ import dask_cudf from pylibcugraph.testing.utils import gen_fixture_params_product -from cugraph.experimental.datasets import DATASETS_UNDIRECTED +from cugraph.datasets import karate, dolphins import cugraph import cugraph.dask as dcg @@ -41,7 +41,7 @@ def setup_function(): # email_Eu_core is too expensive to test -datasets = DATASETS_UNDIRECTED +datasets = [karate, dolphins] # ============================================================================= diff --git a/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py b/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py index d93fa3b547d..8d80611a54c 100644 --- a/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py +++ b/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py @@ -93,7 +93,7 @@ def input_expected_output(input_combo): srcs = G.view_edge_list()["0"] dsts = G.view_edge_list()["1"] vertices = cudf.concat([srcs, dsts]).drop_duplicates() - vertices = vertices.sample(num_seeds).astype("int32") + vertices = vertices.sample(num_seeds, replace=True).astype("int32") # print randomly sample n seeds from the graph print("\nvertices: \n", vertices) diff --git a/python/cugraph/cugraph/tests/community/test_leiden.py b/python/cugraph/cugraph/tests/community/test_leiden.py index a06b0dd22c5..71117c4210f 100644 --- a/python/cugraph/cugraph/tests/community/test_leiden.py +++ b/python/cugraph/cugraph/tests/community/test_leiden.py @@ -22,8 +22,6 @@ from cugraph.testing import utils, UNDIRECTED_DATASETS from cugraph.datasets import karate_asymmetric -from cudf.testing.testing import assert_series_equal - # ============================================================================= # Test data @@ -43,8 +41,8 @@ "resolution": 1.0, "input_type": "COO", "expected_output": { - "partition": [1, 0, 1, 2, 2, 2], - "modularity_score": 0.1757322, + "partition": [0, 0, 0, 1, 1, 1], + "modularity_score": 0.215969, }, }, "data_2": { @@ -85,10 +83,10 @@ "input_type": "CSR", "expected_output": { # fmt: off - "partition": [6, 6, 3, 3, 1, 5, 5, 3, 0, 3, 1, 6, 3, 3, 4, 4, 5, 6, 4, 6, 4, - 6, 4, 4, 2, 2, 4, 4, 2, 4, 0, 2, 4, 4], + "partition": [3, 3, 3, 3, 2, 2, 2, 3, 1, 3, 2, 3, 3, 3, 1, 1, 2, 3, 1, 3, + 1, 3, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1], # fmt: on - "modularity_score": 0.3468113, + "modularity_score": 0.41880345, }, }, } @@ -138,7 +136,7 @@ def input_and_expected_output(request): # Create graph from csr offsets = src_or_offset_array indices = dst_or_index_array - G.from_cudf_adjlist(offsets, indices, weight) + G.from_cudf_adjlist(offsets, indices, weight, renumber=False) parts, mod = cugraph.leiden(G, max_level, resolution) @@ -223,9 +221,7 @@ def test_leiden_directed_graph(): @pytest.mark.sg def test_leiden_golden_results(input_and_expected_output): - expected_partition = cudf.Series( - input_and_expected_output["expected_output"]["partition"] - ) + expected_partition = input_and_expected_output["expected_output"]["partition"] expected_mod = input_and_expected_output["expected_output"]["modularity_score"] result_partition = input_and_expected_output["result_output"]["partition"] @@ -233,6 +229,10 @@ def test_leiden_golden_results(input_and_expected_output): assert abs(expected_mod - result_mod) < 0.0001 - assert_series_equal( - expected_partition, result_partition, check_dtype=False, check_names=False - ) + expected_to_result_map = {} + for e, r in zip(expected_partition, list(result_partition.to_pandas())): + if e in expected_to_result_map.keys(): + assert r == expected_to_result_map[e] + + else: + expected_to_result_map[e] = r diff --git a/python/cugraph/cugraph/tests/conftest.py b/python/cugraph/cugraph/tests/conftest.py index 916e445cfdb..cb5755128eb 100644 --- a/python/cugraph/cugraph/tests/conftest.py +++ b/python/cugraph/cugraph/tests/conftest.py @@ -20,6 +20,9 @@ import os import tempfile +# Avoid timeout during shutdown +from dask_cuda.utils_test import IncreasedCloseTimeoutNanny + # module-wide fixtures @@ -40,7 +43,9 @@ def dask_client(): # start_dask_client will check for the SCHEDULER_FILE and # DASK_WORKER_DEVICES env vars and use them when creating a client if # set. start_dask_client will also initialize the Comms singleton. - dask_client, dask_cluster = start_dask_client() + dask_client, dask_cluster = start_dask_client( + worker_class=IncreasedCloseTimeoutNanny + ) yield dask_client diff --git a/python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py b/python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py new file mode 100644 index 00000000000..3bdb5c079ef --- /dev/null +++ b/python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc + +import pytest + +import dask_cudf +import numpy as np +from cugraph.testing import UNDIRECTED_DATASETS, karate_disjoint + +from cugraph.structure.replicate_edgelist import replicate_edgelist +from cudf.testing.testing import assert_frame_equal +from pylibcugraph.testing.utils import gen_fixture_params_product + + +# ============================================================================= +# Pytest Setup / Teardown - called for each test function +# ============================================================================= +def setup_function(): + gc.collect() + + +edgeWeightCol = "weights" +edgeIdCol = "edge_id" +edgeTypeCol = "edge_type" +srcCol = "src" +dstCol = "dst" + + +input_data = UNDIRECTED_DATASETS + [karate_disjoint] +datasets = [pytest.param(d) for d in input_data] + +fixture_params = gen_fixture_params_product( + (datasets, "graph_file"), + ([True, False], "distributed"), + ([True, False], "use_weights"), + ([True, False], "use_edge_ids"), + ([True, False], "use_edge_type_ids"), +) + + +@pytest.fixture(scope="module", params=fixture_params) +def input_combo(request): + """ + Simply return the current combination of params as a dictionary for use in + tests or other parameterized fixtures. + """ + return dict( + zip( + ( + "graph_file", + "use_weights", + "use_edge_ids", + "use_edge_type_ids", + "distributed", + ), + request.param, + ) + ) + + +# ============================================================================= +# Tests +# ============================================================================= +# @pytest.mark.skipif( +# is_single_gpu(), reason="skipping MG testing on Single GPU system" +# ) +@pytest.mark.mg +def test_mg_replicate_edgelist(dask_client, input_combo): + df = input_combo["graph_file"].get_edgelist() + distributed = input_combo["distributed"] + + use_weights = input_combo["use_weights"] + use_edge_ids = input_combo["use_edge_ids"] + use_edge_type_ids = input_combo["use_edge_type_ids"] + + columns = [srcCol, dstCol] + weight = None + edge_id = None + edge_type = None + + if use_weights: + df = df.rename(columns={"wgt": edgeWeightCol}) + columns.append(edgeWeightCol) + weight = edgeWeightCol + if use_edge_ids: + df = df.reset_index().rename(columns={"index": edgeIdCol}) + df[edgeIdCol] = df[edgeIdCol].astype(df[srcCol].dtype) + columns.append(edgeIdCol) + edge_id = edgeIdCol + if use_edge_type_ids: + df[edgeTypeCol] = np.random.randint(0, 10, size=len(df)) + df[edgeTypeCol] = df[edgeTypeCol].astype(df[srcCol].dtype) + columns.append(edgeTypeCol) + edge_type = edgeTypeCol + + if distributed: + # Distribute the edges across all ranks + num_workers = len(dask_client.scheduler_info()["workers"]) + df = dask_cudf.from_cudf(df, npartitions=num_workers) + ddf = replicate_edgelist( + df[columns], weight=weight, edge_id=edge_id, edge_type=edge_type + ) + + if distributed: + df = df.compute() + + for i in range(ddf.npartitions): + result_df = ( + ddf.get_partition(i) + .compute() + .sort_values([srcCol, dstCol]) + .reset_index(drop=True) + ) + expected_df = df[columns].sort_values([srcCol, dstCol]).reset_index(drop=True) + + assert_frame_equal(expected_df, result_df, check_dtype=False, check_like=True) diff --git a/python/cugraph/cugraph/tests/link_analysis/test_hits.py b/python/cugraph/cugraph/tests/link_analysis/test_hits.py index 1c5a135e944..fcfd8cc5318 100644 --- a/python/cugraph/cugraph/tests/link_analysis/test_hits.py +++ b/python/cugraph/cugraph/tests/link_analysis/test_hits.py @@ -38,7 +38,11 @@ def setup_function(): fixture_params = gen_fixture_params_product( (datasets, "graph_file"), ([50], "max_iter"), - ([1.0e-6], "tol"), + # FIXME: Changed this from 1.0e-6 to 1.0e-5. NX defaults to + # FLOAT64 computation, cuGraph C++ defaults to whatever the edge weight + # is, cugraph python defaults that to FLOAT32. Does not converge at + # 1e-6 for larger graphs and FLOAT32. + ([1.0e-5], "tol"), ) diff --git a/python/cugraph/cugraph/tests/link_analysis/test_hits_mg.py b/python/cugraph/cugraph/tests/link_analysis/test_hits_mg.py index 5590eb17401..73ec13c674c 100644 --- a/python/cugraph/cugraph/tests/link_analysis/test_hits_mg.py +++ b/python/cugraph/cugraph/tests/link_analysis/test_hits_mg.py @@ -45,7 +45,7 @@ def setup_function(): fixture_params = gen_fixture_params_product( (datasets, "graph_file"), ([50], "max_iter"), - ([1.0e-6], "tol"), + ([1.0e-4], "tol"), # FIXME: Temporarily lower tolerance (IS_DIRECTED, "directed"), ) diff --git a/python/cugraph/cugraph/tests/nx/test_compat_pr.py b/python/cugraph/cugraph/tests/nx/test_compat_pr.py index 9be3912a33f..45cab7a5674 100644 --- a/python/cugraph/cugraph/tests/nx/test_compat_pr.py +++ b/python/cugraph/cugraph/tests/nx/test_compat_pr.py @@ -24,7 +24,7 @@ import numpy as np from cugraph.testing import utils -from cugraph.experimental.datasets import karate +from cugraph.datasets import karate from pylibcugraph.testing.utils import gen_fixture_params_product diff --git a/python/cugraph/cugraph/tests/structure/test_graph_mg.py b/python/cugraph/cugraph/tests/structure/test_graph_mg.py index 3024e50402a..7837916ae53 100644 --- a/python/cugraph/cugraph/tests/structure/test_graph_mg.py +++ b/python/cugraph/cugraph/tests/structure/test_graph_mg.py @@ -30,6 +30,9 @@ from cugraph.dask.traversal.bfs import convert_to_cudf from cugraph.dask.common.input_utils import get_distributed_data from pylibcugraph.testing.utils import gen_fixture_params_product +from cugraph.dask.common.part_utils import ( + persist_dask_df_equal_parts_per_worker, +) # ============================================================================= @@ -141,10 +144,13 @@ def test_create_mg_graph(dask_client, input_combo): assert len(G._plc_graph) == len(dask_client.has_what()) start = dask_cudf.from_cudf(cudf.Series([1], dtype="int32"), len(G._plc_graph)) + vertex_dtype = start.dtype if G.renumbered: start = G.lookup_internal_vertex_id(start, None) - data_start = get_distributed_data(start) + data_start = persist_dask_df_equal_parts_per_worker( + start, dask_client, return_type="dict" + ) res = [ dask_client.submit( @@ -159,10 +165,10 @@ def test_create_mg_graph(dask_client, input_combo): ), Comms.get_session_id(), G._plc_graph[w], - data_start.worker_to_parts[w][0], + st[0] if st else cudf.Series(dtype=vertex_dtype), workers=[w], ) - for w in Comms.get_workers() + for w, st in data_start.items() ] wait(res) diff --git a/python/cugraph/cugraph/tests/utils/test_dataset.py b/python/cugraph/cugraph/tests/utils/test_dataset.py index c2a4f7c6072..60bc6dbb45a 100644 --- a/python/cugraph/cugraph/tests/utils/test_dataset.py +++ b/python/cugraph/cugraph/tests/utils/test_dataset.py @@ -13,11 +13,10 @@ import os import gc -import sys -import warnings from pathlib import Path from tempfile import TemporaryDirectory +import pandas import pytest import cudf @@ -27,6 +26,7 @@ ALL_DATASETS, WEIGHTED_DATASETS, SMALL_DATASETS, + BENCHMARKING_DATASETS, ) from cugraph import datasets @@ -74,27 +74,14 @@ def setup(tmpdir): gc.collect() -@pytest.fixture() -def setup_deprecation_warning_tests(): - """ - Fixture used to set warning filters to 'default' and reload - experimental.datasets module if it has been previously - imported. Tests that import this fixture are expected to - import cugraph.experimental.datasets - """ - warnings.filterwarnings("default") - - if "cugraph.experimental.datasets" in sys.modules: - del sys.modules["cugraph.experimental.datasets"] - - yield - - ############################################################################### # Helpers # check if there is a row where src == dst -def has_loop(df): +def has_selfloop(dataset): + if not dataset.metadata["is_directed"]: + return False + df = dataset.get_edgelist(download=True) df.rename(columns={df.columns[0]: "src", df.columns[1]: "dst"}, inplace=True) res = df.where(df["src"] == df["dst"]) @@ -109,7 +96,13 @@ def is_symmetric(dataset): else: df = dataset.get_edgelist(download=True) df_a = df.sort_values("src") - df_b = df_a[["dst", "src", "wgt"]] + + # create df with swapped src/dst columns + df_b = None + if "wgt" in df_a.columns: + df_b = df_a[["dst", "src", "wgt"]] + else: + df_b = df_a[["dst", "src"]] df_b.rename(columns={"dst": "src", "src": "dst"}, inplace=True) # created a df by appending the two res = cudf.concat([df_a, df_b]) @@ -157,6 +150,27 @@ def test_download(dataset): assert dataset.get_path().is_file() +@pytest.mark.parametrize("dataset", SMALL_DATASETS) +def test_reader(dataset): + # defaults to using cudf.read_csv + E = dataset.get_edgelist(download=True) + + assert E is not None + assert isinstance(E, cudf.core.dataframe.DataFrame) + dataset.unload() + + # using pandas + E_pd = dataset.get_edgelist(download=True, reader="pandas") + + assert E_pd is not None + assert isinstance(E_pd, pandas.core.frame.DataFrame) + dataset.unload() + + with pytest.raises(ValueError): + dataset.get_edgelist(reader="fail") + dataset.get_edgelist(reader=None) + + @pytest.mark.parametrize("dataset", ALL_DATASETS) def test_get_edgelist(dataset): E = dataset.get_edgelist(download=True) @@ -172,7 +186,6 @@ def test_get_graph(dataset): @pytest.mark.parametrize("dataset", ALL_DATASETS) def test_metadata(dataset): M = dataset.metadata - assert M is not None @@ -310,10 +323,8 @@ def test_is_directed(dataset): @pytest.mark.parametrize("dataset", ALL_DATASETS) -def test_has_loop(dataset): - df = dataset.get_edgelist(download=True) - - assert has_loop(df) == dataset.metadata["has_loop"] +def test_has_selfloop(dataset): + assert has_selfloop(dataset) == dataset.metadata["has_loop"] @pytest.mark.parametrize("dataset", ALL_DATASETS) @@ -328,6 +339,25 @@ def test_is_multigraph(dataset): assert G.is_multigraph() == dataset.metadata["is_multigraph"] +# The datasets used for benchmarks are in their own test, since downloading them +# repeatedly would increase testing overhead significantly +@pytest.mark.parametrize("dataset", BENCHMARKING_DATASETS) +def test_benchmarking_datasets(dataset): + dataset_is_directed = dataset.metadata["is_directed"] + G = dataset.get_graph( + download=True, create_using=Graph(directed=dataset_is_directed) + ) + + assert G.is_directed() == dataset.metadata["is_directed"] + assert G.number_of_nodes() == dataset.metadata["number_of_nodes"] + assert G.number_of_edges() == dataset.metadata["number_of_edges"] + assert has_selfloop(dataset) == dataset.metadata["has_loop"] + assert is_symmetric(dataset) == dataset.metadata["is_symmetric"] + assert G.is_multigraph() == dataset.metadata["is_multigraph"] + + dataset.unload() + + @pytest.mark.parametrize("dataset", ALL_DATASETS) def test_object_getters(dataset): assert dataset.is_directed() == dataset.metadata["is_directed"] @@ -336,32 +366,3 @@ def test_object_getters(dataset): assert dataset.number_of_nodes() == dataset.metadata["number_of_nodes"] assert dataset.number_of_vertices() == dataset.metadata["number_of_nodes"] assert dataset.number_of_edges() == dataset.metadata["number_of_edges"] - - -# -# Test experimental for DeprecationWarnings -# -def test_experimental_dataset_import(setup_deprecation_warning_tests): - with pytest.deprecated_call(): - from cugraph.experimental.datasets import karate - - # unload() is called to pass flake8 - karate.unload() - - -def test_experimental_method_warnings(setup_deprecation_warning_tests): - from cugraph.experimental.datasets import ( - load_all, - set_download_dir, - get_download_dir, - ) - - warnings.filterwarnings("default") - tmpd = TemporaryDirectory() - - with pytest.deprecated_call(): - set_download_dir(tmpd.name) - get_download_dir() - load_all() - - tmpd.cleanup() diff --git a/python/cugraph/cugraph/tests/utils/test_resultset.py b/python/cugraph/cugraph/tests/utils/test_resultset.py new file mode 100644 index 00000000000..5c2298bedb7 --- /dev/null +++ b/python/cugraph/cugraph/tests/utils/test_resultset.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from pathlib import Path +from tempfile import TemporaryDirectory + +import cudf +from cugraph.datasets.dataset import ( + set_download_dir, + get_download_dir, +) +from cugraph.testing.resultset import load_resultset, default_resultset_download_dir + +############################################################################### + + +def test_load_resultset(): + with TemporaryDirectory() as tmpd: + + set_download_dir(Path(tmpd)) + default_resultset_download_dir.path = Path(tmpd) / "tests" / "resultsets" + default_resultset_download_dir.path.mkdir(parents=True, exist_ok=True) + + datasets_download_dir = get_download_dir() + resultsets_download_dir = default_resultset_download_dir.path + assert "tests" in os.listdir(datasets_download_dir) + assert "resultsets.tar.gz" not in os.listdir(datasets_download_dir / "tests") + assert "traversal_mappings.csv" not in os.listdir(resultsets_download_dir) + + load_resultset( + "traversal", "https://data.rapids.ai/cugraph/results/resultsets.tar.gz" + ) + + assert "resultsets.tar.gz" in os.listdir(datasets_download_dir / "tests") + assert "traversal_mappings.csv" in os.listdir(resultsets_download_dir) + + +def test_verify_resultset_load(): + # This test is more detailed than test_load_resultset, where for each module, + # we check that every single resultset file is included along with the + # corresponding mapping file. + with TemporaryDirectory() as tmpd: + set_download_dir(Path(tmpd)) + default_resultset_download_dir.path = Path(tmpd) / "tests" / "resultsets" + default_resultset_download_dir.path.mkdir(parents=True, exist_ok=True) + + resultsets_download_dir = default_resultset_download_dir.path + + load_resultset( + "traversal", "https://data.rapids.ai/cugraph/results/resultsets.tar.gz" + ) + + resultsets = os.listdir(resultsets_download_dir) + downloaded_results = cudf.read_csv( + resultsets_download_dir / "traversal_mappings.csv", sep=" " + ) + downloaded_uuids = downloaded_results["#UUID"].values + for resultset_uuid in downloaded_uuids: + assert str(resultset_uuid) + ".csv" in resultsets diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 1835ac8bb49..bd426291c8d 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -6,9 +6,9 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", "ninja", - "pylibcugraph==23.10.*", - "pylibraft==23.10.*", - "rmm==23.10.*", + "pylibcugraph==23.12.*", + "pylibraft==23.12.*", + "rmm==23.12.*", "scikit-build>=0.13.1", "setuptools>=61.0.0", "wheel", @@ -20,7 +20,7 @@ testpaths = ["cugraph/tests"] [project] name = "cugraph" -version = "23.10.00" +dynamic = ["version"] description = "cuGraph - RAPIDS GPU Graph Analytics" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -29,18 +29,18 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cudf==23.10.*", + "cudf==23.12.*", "cupy-cuda11x>=12.0.0", - "dask-cuda==23.10.*", - "dask-cudf==23.10.*", - "dask==2023.9.2", - "distributed==2023.9.2", + "dask-cuda==23.12.*", + "dask-cudf==23.12.*", "fsspec[http]>=0.6.0", "numba>=0.57", - "pylibcugraph==23.10.*", - "raft-dask==23.10.*", - "rmm==23.10.*", - "ucx-py==0.34.*", + "numpy>=1.21", + "pylibcugraph==23.12.*", + "raft-dask==23.12.*", + "rapids-dask-dependency==23.12.*", + "rmm==23.12.*", + "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -69,3 +69,6 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [tool.setuptools] license-files = ["LICENSE"] + +[tool.setuptools.dynamic] +version = {file = "cugraph/VERSION"} diff --git a/python/cugraph/setup.py b/python/cugraph/setup.py index aa3a5fb56a7..81916444cfd 100644 --- a/python/cugraph/setup.py +++ b/python/cugraph/setup.py @@ -46,7 +46,7 @@ def run(self): packages = find_packages(include=["cugraph*"]) setup( packages=packages, - package_data={key: ["*.pxd", "*.yaml"] for key in packages}, + package_data={key: ["VERSION", "*.pxd", "*.yaml"] for key in packages}, cmdclass={"clean": CleanCommand}, zip_safe=False, ) diff --git a/python/nx-cugraph/README.md b/python/nx-cugraph/README.md index e7cd26218e6..f6a9aac1088 100644 --- a/python/nx-cugraph/README.md +++ b/python/nx-cugraph/README.md @@ -1,32 +1,136 @@ # nx-cugraph ## Description -[RAPIDS](https://rapids.ai) nx-cugraph is a [backend to NetworkX](https://networkx.org/documentation/stable/reference/classes/index.html#backends) -with minimal dependencies (`networkx`, `cupy`, and `pylibcugraph`) to run graph algorithms on the GPU. +[RAPIDS](https://rapids.ai) nx-cugraph is a [backend to NetworkX](https://networkx.org/documentation/stable/reference/utils.html#backends) +to run supported algorithms with GPU acceleration. -### Contribute +## System Requirements -Follow instructions for [contributing to cugraph](https://github.com/rapidsai/cugraph/blob/branch-23.10/readme_pages/CONTRIBUTING.md) -and [building from source](https://docs.rapids.ai/api/cugraph/stable/installation/source_build/), then build nx-cugraph in develop (i.e., editable) mode: -``` -$ ./build.sh nx-cugraph --pydevelop -``` +nx-cugraph requires the following: + + * NVIDIA GPU, Pascal architecture or later + * CUDA 11.2, 11.4, 11.5, 11.8, or 12.0 + * Python versions 3.9, 3.10, or 3.11 + * NetworkX >= version 3.2 -### Run tests +More details about system requirements can be found in the [RAPIDS System Requirements documentation](https://docs.rapids.ai/install#system-req). -Run nx-cugraph tests from `cugraph/python/nx-cugraph` directory: +## Installation + +nx-cugraph can be installed using either conda or pip. + +### conda ``` -$ pytest +conda install -c rapidsai-nightly -c conda-forge -c nvidia nx-cugraph ``` -Run nx-cugraph benchmarks: +### pip ``` -$ pytest --bench +python -m pip install nx-cugraph-cu11 --extra-index-url https://pypi.nvidia.com ``` -Run networkx tests (requires networkx version 3.2): +Notes: + + * Nightly wheel builds will not be available until the 23.12 release, therefore the index URL for the stable release version is being used in the pip install command above. + * Additional information relevant to installing any RAPIDS package can be found [here](https://rapids.ai/#quick-start). + +## Enabling nx-cugraph + +NetworkX will use nx-cugraph as the graph analytics backend if any of the +following are used: + +### `NETWORKX_AUTOMATIC_BACKENDS` environment variable. +The `NETWORKX_AUTOMATIC_BACKENDS` environment variable can be used to have NetworkX automatically dispatch to specified backends an API is called that the backend supports. +Set `NETWORKX_AUTOMATIC_BACKENDS=cugraph` to use nx-cugraph to GPU accelerate supported APIs with no code changes. +Example: ``` -$ ./run_nx_tests.sh +bash> NETWORKX_AUTOMATIC_BACKENDS=cugraph python my_networkx_script.py ``` -Additional arguments may be passed to pytest such as: + +### `backend=` keyword argument +To explicitly specify a particular backend for an API, use the `backend=` +keyword argument. This argument takes precedence over the +`NETWORKX_AUTOMATIC_BACKENDS` environment variable. This requires anyone +running code that uses the `backend=` keyword argument to have the specified +backend installed. + +Example: ``` -$ ./run_nx_tests.sh -x --sw -k betweenness +nx.betweenness_centrality(cit_patents_graph, k=k, backend="cugraph") ``` + +### Type-based dispatching + +NetworkX also supports automatically dispatching to backends associated with +specific graph types. Like the `backend=` keyword argument example above, this +requires the user to write code for a specific backend, and therefore requires +the backend to be installed, but has the advantage of ensuring a particular +behavior without the potential for runtime conversions. + +To use type-based dispatching with nx-cugraph, the user must import the backend +directly in their code to access the utilities provided to create a Graph +instance specifically for the nx-cugraph backend. + +Example: +``` +import networkx as nx +import nx_cugraph as nxcg + +G = nx.Graph() +... +nxcg_G = nxcg.from_networkx(G) # conversion happens once here +nx.betweenness_centrality(nxcg_G, k=1000) # nxcg Graph type causes cugraph backend + # to be used, no conversion necessary +``` + +## Supported Algorithms + +The nx-cugraph backend to NetworkX connects +[pylibcugraph](../../readme_pages/pylibcugraph.md) (cuGraph's low-level python +interface to its CUDA-based graph analytics library) and +[CuPy](https://cupy.dev/) (a GPU-accelerated array library) to NetworkX's +familiar and easy-to-use API. + +Below is the list of algorithms (many listed using pylibcugraph names), +available today in pylibcugraph or implemented using CuPy, that are or will be +supported in nx-cugraph. + +| feature/algo | release/target version | +| ----- | ----- | +| analyze_clustering_edge_cut | ? | +| analyze_clustering_modularity | ? | +| analyze_clustering_ratio_cut | ? | +| balanced_cut_clustering | ? | +| betweenness_centrality | 23.10 | +| bfs | ? | +| connected_components | 23.12 | +| core_number | ? | +| degree_centrality | 23.12 | +| ecg | ? | +| edge_betweenness_centrality | 23.10 | +| ego_graph | ? | +| eigenvector_centrality | 23.12 | +| get_two_hop_neighbors | ? | +| hits | 23.12 | +| in_degree_centrality | 23.12 | +| induced_subgraph | ? | +| jaccard_coefficients | ? | +| katz_centrality | 23.12 | +| k_core | ? | +| k_truss_subgraph | 23.12 | +| leiden | ? | +| louvain | 23.10 | +| node2vec | ? | +| out_degree_centrality | 23.12 | +| overlap_coefficients | ? | +| pagerank | 23.12 | +| personalized_pagerank | ? | +| sorensen_coefficients | ? | +| spectral_modularity_maximization | ? | +| sssp | 23.12 | +| strongly_connected_components | ? | +| triangle_count | ? | +| uniform_neighbor_sample | ? | +| uniform_random_walks | ? | +| weakly_connected_components | ? | + +To request nx-cugraph backend support for a NetworkX API that is not listed +above, visit the [cuGraph GitHub repo](https://github.com/rapidsai/cugraph). diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index ebd13daded0..ef5f8f3fc23 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -24,31 +24,112 @@ "backend_name": "cugraph", "project": "nx-cugraph", "package": "nx_cugraph", - "url": "https://github.com/rapidsai/cugraph/tree/branch-23.10/python/nx-cugraph", + "url": "https://github.com/rapidsai/cugraph/tree/branch-23.12/python/nx-cugraph", "short_summary": "GPU-accelerated backend.", # "description": "TODO", "functions": { # BEGIN: functions + "barbell_graph", "betweenness_centrality", + "bull_graph", + "caveman_graph", + "chvatal_graph", + "circular_ladder_graph", + "complete_bipartite_graph", + "complete_graph", + "complete_multipartite_graph", + "connected_components", + "cubical_graph", + "cycle_graph", + "davis_southern_women_graph", + "degree_centrality", + "desargues_graph", + "diamond_graph", + "dodecahedral_graph", "edge_betweenness_centrality", + "eigenvector_centrality", + "empty_graph", + "florentine_families_graph", + "from_pandas_edgelist", + "from_scipy_sparse_array", + "frucht_graph", + "heawood_graph", + "hits", + "house_graph", + "house_x_graph", + "icosahedral_graph", + "in_degree_centrality", + "is_connected", "is_isolate", "isolates", + "k_truss", + "karate_club_graph", + "katz_centrality", + "krackhardt_kite_graph", + "ladder_graph", + "les_miserables_graph", + "lollipop_graph", "louvain_communities", + "moebius_kantor_graph", + "node_connected_component", + "null_graph", + "number_connected_components", "number_of_isolates", + "number_of_selfloops", + "octahedral_graph", + "out_degree_centrality", + "pagerank", + "pappus_graph", + "path_graph", + "petersen_graph", + "sedgewick_maze_graph", + "single_source_shortest_path_length", + "single_target_shortest_path_length", + "star_graph", + "tadpole_graph", + "tetrahedral_graph", + "trivial_graph", + "truncated_cube_graph", + "truncated_tetrahedron_graph", + "turan_graph", + "tutte_graph", + "wheel_graph", # END: functions }, "extra_docstrings": { # BEGIN: extra_docstrings "betweenness_centrality": "`weight` parameter is not yet supported.", "edge_betweenness_centrality": "`weight` parameter is not yet supported.", + "eigenvector_centrality": "`nstart` parameter is not used, but it is checked for validity.", + "from_pandas_edgelist": "cudf.DataFrame inputs also supported.", + "k_truss": ( + "Currently raises `NotImplementedError` for graphs with more than one connected\n" + "component when k >= 3. We expect to fix this soon." + ), + "katz_centrality": "`nstart` isn't used (but is checked), and `normalized=False` is not supported.", "louvain_communities": "`seed` parameter is currently ignored.", + "pagerank": "`dangling` parameter is not supported, but it is checked for validity.", # END: extra_docstrings }, "extra_parameters": { # BEGIN: extra_parameters + "eigenvector_centrality": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + }, + "hits": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + 'weight : string or None, optional (default="weight")': "The edge attribute to use as the edge weight.", + }, + "katz_centrality": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + }, "louvain_communities": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", "max_level : int, optional": "Upper limit of the number of macro-iterations (max: 500).", }, + "pagerank": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + }, # END: extra_parameters }, } @@ -78,7 +159,8 @@ def get_info(): return d -__version__ = "23.10.00" +# FIXME: can this use the standard VERSION file and update mechanism? +__version__ = "23.12.00" if __name__ == "__main__": from pathlib import Path diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index 338ca7b230e..a94aa9f0448 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -11,7 +11,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -26,7 +26,7 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.14 + rev: v0.15 hooks: - id: validate-pyproject name: Validate pyproject.toml @@ -40,25 +40,25 @@ repos: hooks: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: v3.13.0 + rev: v3.15.0 hooks: - id: pyupgrade args: [--py39-plus] - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.1 hooks: - id: black # - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.291 + rev: v0.1.3 hooks: - id: ruff - args: [--fix-only, --show-fixes] + args: [--fix-only, --show-fixes] # --unsafe-fixes] - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: - id: flake8 - args: ['--per-file-ignores=_nx_cugraph/__init__.py:E501'] # Why is this necessary? + args: ['--per-file-ignores=_nx_cugraph/__init__.py:E501', '--extend-ignore=SIM105'] # Why is this necessary? additional_dependencies: &flake8_dependencies # These versions need updated manually - flake8==6.1.0 @@ -70,18 +70,18 @@ repos: - id: yesqa additional_dependencies: *flake8_dependencies - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell types_or: [python, rst, markdown] additional_dependencies: [tomli] files: ^(nx_cugraph|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.291 + rev: v0.1.3 hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: no-commit-to-branch args: [-p, "^branch-2....$"] diff --git a/python/nx-cugraph/nx_cugraph/VERSION b/python/nx-cugraph/nx_cugraph/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 4a0e95a109f..3a8f0996e9c 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -20,13 +20,13 @@ from . import convert from .convert import * -# from . import convert_matrix -# from .convert_matrix import * +from . import convert_matrix +from .convert_matrix import * -# from . import generators -# from .generators import * +from . import generators +from .generators import * from . import algorithms from .algorithms import * -__version__ = "23.10.00" +from nx_cugraph._version import __git_commit__, __version__ diff --git a/python/nx-cugraph/nx_cugraph/_version.py b/python/nx-cugraph/nx_cugraph/_version.py new file mode 100644 index 00000000000..868a2e19475 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/_version.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("nx_cugraph").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/nx-cugraph/nx_cugraph/algorithms/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/__init__.py index dfd9adfc61a..63841b15bd5 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/__init__.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/__init__.py @@ -10,6 +10,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from . import centrality, community +from . import ( + bipartite, + centrality, + community, + components, + shortest_paths, + link_analysis, +) +from .bipartite import complete_bipartite_graph from .centrality import * +from .components import * +from .core import * from .isolate import * +from .shortest_paths import * +from .link_analysis import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/__init__.py new file mode 100644 index 00000000000..062be973d55 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .generators import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py new file mode 100644 index 00000000000..1d3e762b4fd --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from numbers import Integral + +import cupy as cp +import networkx as nx +import numpy as np + +from nx_cugraph.generators._utils import _create_using_class, _number_and_nodes +from nx_cugraph.utils import index_dtype, networkx_algorithm, nodes_or_number + +__all__ = [ + "complete_bipartite_graph", +] + + +@nodes_or_number([0, 1]) +@networkx_algorithm +def complete_bipartite_graph(n1, n2, create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + orig_n1, unused_nodes1 = n1 + orig_n2, unused_nodes2 = n2 + n1, nodes1 = _number_and_nodes(n1) + n2, nodes2 = _number_and_nodes(n2) + all_indices = cp.indices((n1, n2), dtype=index_dtype) + indices0 = all_indices[0].ravel() + indices1 = all_indices[1].ravel() + n1 + del all_indices + src_indices = cp.hstack((indices0, indices1)) + dst_indices = cp.hstack((indices1, indices0)) + bipartite = cp.zeros(n1 + n2, np.int8) + bipartite[n1:] = 1 + if isinstance(orig_n1, Integral) and isinstance(orig_n2, Integral): + nodes = None + else: + nodes = list(range(n1)) if nodes1 is None else nodes1 + nodes.extend(range(n2) if nodes2 is None else nodes2) + if len(set(nodes)) != len(nodes): + raise nx.NetworkXError("Inputs n1 and n2 must contain distinct nodes") + G = graph_class.from_coo( + n1 + n2, + src_indices, + dst_indices, + node_values={"bipartite": bipartite}, + id_to_key=nodes, + name=f"complete_bipartite_graph({orig_n1}, {orig_n2})", + ) + if inplace: + return create_using._become(G) + return G diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py index 2ac6242e8a4..496dc6aff81 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py @@ -11,3 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. from .betweenness import * +from .degree_alg import * +from .eigenvector import * +from .katz import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py index 104ac87414c..210e1f0a2b2 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py @@ -53,6 +53,7 @@ def edge_betweenness_centrality(G, k=None, normalized=True, weight=None, seed=No raise NotImplementedError( "Weighted implementation of betweenness centrality not currently supported" ) + seed = _seed_to_int(seed) G = _to_graph(G, weight) src_ids, dst_ids, values, _edge_ids = plc.edge_betweenness_centrality( resource_handle=plc.ResourceHandle(), diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/degree_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/degree_alg.py new file mode 100644 index 00000000000..0b2fd24af79 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/degree_alg.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nx_cugraph.convert import _to_directed_graph, _to_graph +from nx_cugraph.utils import networkx_algorithm, not_implemented_for + +__all__ = ["degree_centrality", "in_degree_centrality", "out_degree_centrality"] + + +@networkx_algorithm +def degree_centrality(G): + G = _to_graph(G) + if len(G) <= 1: + return dict.fromkeys(G, 1) + deg = G._degrees_array() + centrality = deg * (1 / (len(G) - 1)) + return G._nodearray_to_dict(centrality) + + +@not_implemented_for("undirected") +@networkx_algorithm +def in_degree_centrality(G): + G = _to_directed_graph(G) + if len(G) <= 1: + return dict.fromkeys(G, 1) + deg = G._in_degrees_array() + centrality = deg * (1 / (len(G) - 1)) + return G._nodearray_to_dict(centrality) + + +@not_implemented_for("undirected") +@networkx_algorithm +def out_degree_centrality(G): + G = _to_directed_graph(G) + if len(G) <= 1: + return dict.fromkeys(G, 1) + deg = G._out_degrees_array() + centrality = deg * (1 / (len(G) - 1)) + return G._nodearray_to_dict(centrality) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py new file mode 100644 index 00000000000..c0f02a6258e --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + networkx_algorithm, + not_implemented_for, +) + +__all__ = ["eigenvector_centrality"] + + +@not_implemented_for("multigraph") +@networkx_algorithm(extra_params=_dtype_param) +def eigenvector_centrality( + G, max_iter=100, tol=1.0e-6, nstart=None, weight=None, *, dtype=None +): + """`nstart` parameter is not used, but it is checked for validity.""" + G = _to_graph(G, weight, np.float32) + if len(G) == 0: + raise nx.NetworkXPointlessConcept( + "cannot compute centrality for the null graph" + ) + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + # Check if given nstart is valid even though we don't use it + nstart = G._dict_to_nodearray(nstart, dtype=dtype) + if (nstart == 0).all(): + raise nx.NetworkXError("initial vector cannot have all zero values") + if nstart.sum() == 0: + raise ZeroDivisionError + # nstart /= total # Uncomment (and assign total) when nstart is used below + try: + node_ids, values = plc.eigenvector_centrality( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(weight, 1, dtype, store_transposed=True), + epsilon=tol, + max_iterations=max_iter, + do_expensive_check=False, + ) + except RuntimeError as exc: + # Errors from PLC are sometimes a little scary and not very helpful + raise nx.PowerIterationFailedConvergence(max_iter) from exc + return G._nodearrays_to_dict(node_ids, values) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py new file mode 100644 index 00000000000..b61b811b8fa --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py @@ -0,0 +1,100 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + networkx_algorithm, + not_implemented_for, +) + +__all__ = ["katz_centrality"] + + +@not_implemented_for("multigraph") +@networkx_algorithm(extra_params=_dtype_param) +def katz_centrality( + G, + alpha=0.1, + beta=1.0, + max_iter=1000, + tol=1.0e-6, + nstart=None, + normalized=True, + weight=None, + *, + dtype=None, +): + """`nstart` isn't used (but is checked), and `normalized=False` is not supported.""" + if not normalized: + # Redundant with the `_can_run` check below when being dispatched by NetworkX, + # but we raise here in case this funcion is called directly. + raise NotImplementedError("normalized=False is not supported.") + G = _to_graph(G, weight, np.float32) + if (N := len(G)) == 0: + return {} + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + # Check if given nstart is valid even though we don't use it + nstart = G._dict_to_nodearray(nstart, 0, dtype) + b = bs = None + try: + b = float(beta) + except (TypeError, ValueError) as exc: + try: + bs = G._dict_to_nodearray(beta, dtype=dtype) + b = 1.0 # float value must be given to PLC (and will be ignored) + except (KeyError, ValueError): + raise nx.NetworkXError( + "beta dictionary must have a value for every node" + ) from exc + try: + node_ids, values = plc.katz_centrality( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(weight, 1, dtype, store_transposed=True), + betas=bs, + alpha=alpha, + beta=b, + epsilon=N * tol, + max_iterations=max_iter, + do_expensive_check=False, + ) + except RuntimeError as exc: + # Errors from PLC are sometimes a little scary and not very helpful + raise nx.PowerIterationFailedConvergence(max_iter) from exc + return G._nodearrays_to_dict(node_ids, values) + + +@katz_centrality._can_run +def _( + G, + alpha=0.1, + beta=1.0, + max_iter=1000, + tol=1.0e-6, + nstart=None, + normalized=True, + weight=None, + *, + dtype=None, +): + return normalized diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index dc209870c89..936d837dacd 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -16,6 +16,7 @@ from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( + _dtype_param, _groupby, _seed_to_int, networkx_algorithm, @@ -32,17 +33,25 @@ extra_params={ "max_level : int, optional": ( "Upper limit of the number of macro-iterations (max: 500)." - ) + ), + **_dtype_param, } ) def louvain_communities( - G, weight="weight", resolution=1, threshold=0.0000001, seed=None, *, max_level=None + G, + weight="weight", + resolution=1, + threshold=0.0000001, + seed=None, + *, + max_level=None, + dtype=None, ): """`seed` parameter is currently ignored.""" # NetworkX allows both directed and undirected, but cugraph only allows undirected. seed = _seed_to_int(seed) # Unused, but ensure it's valid for future compatibility G = _to_undirected_graph(G, weight) - if G.row_indices.size == 0: + if G.src_indices.size == 0: # TODO: PLC doesn't handle empty graphs gracefully! return [{key} for key in G._nodeiter_to_iter(range(len(G)))] if max_level is None: @@ -54,20 +63,20 @@ def louvain_communities( stacklevel=2, ) max_level = 500 - vertices, clusters, modularity = plc.louvain( + node_ids, clusters, modularity = plc.louvain( resource_handle=plc.ResourceHandle(), - graph=G._get_plc_graph(), + graph=G._get_plc_graph(weight, 1, dtype), max_level=max_level, # TODO: add this parameter to NetworkX threshold=threshold, resolution=resolution, do_expensive_check=False, ) - groups = _groupby(clusters, vertices) - rv = [set(G._nodearray_to_list(node_ids)) for node_ids in groups.values()] - # TODO: PLC doesn't handle isolated vertices yet, so this is a temporary fix + groups = _groupby(clusters, node_ids, groups_are_canonical=True) + rv = [set(G._nodearray_to_list(ids)) for ids in groups.values()] + # TODO: PLC doesn't handle isolated node_ids yet, so this is a temporary fix isolates = _isolates(G) if isolates.size > 0: - isolates = isolates[isolates > vertices.max()] + isolates = isolates[isolates > node_ids.max()] if isolates.size > 0: rv.extend({node} for node in G._nodearray_to_list(isolates)) return rv @@ -75,7 +84,14 @@ def louvain_communities( @louvain_communities._can_run def _( - G, weight="weight", resolution=1, threshold=0.0000001, seed=None, *, max_level=None + G, + weight="weight", + resolution=1, + threshold=0.0000001, + seed=None, + *, + max_level=None, + dtype=None, ): # NetworkX allows both directed and undirected, but cugraph only allows undirected. return not G.is_directed() diff --git a/python/cugraph/cugraph/experimental/datasets/metadata/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/components/__init__.py similarity index 89% rename from python/cugraph/cugraph/experimental/datasets/metadata/__init__.py rename to python/nx-cugraph/nx_cugraph/algorithms/components/__init__.py index 081b2ae8260..26816ef3692 100644 --- a/python/cugraph/cugraph/experimental/datasets/metadata/__init__.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/components/__init__.py @@ -1,5 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# +# Copyright (c) 2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,3 +10,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .connected import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/components/connected.py b/python/nx-cugraph/nx_cugraph/algorithms/components/connected.py new file mode 100644 index 00000000000..41f3457d542 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/components/connected.py @@ -0,0 +1,130 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import itertools + +import cupy as cp +import networkx as nx +import pylibcugraph as plc + +from nx_cugraph.convert import _to_undirected_graph +from nx_cugraph.utils import _groupby, networkx_algorithm, not_implemented_for + +from ..isolate import _isolates + +__all__ = [ + "number_connected_components", + "connected_components", + "is_connected", + "node_connected_component", +] + + +@not_implemented_for("directed") +@networkx_algorithm +def number_connected_components(G): + return sum(1 for _ in connected_components(G)) + # PREFERRED IMPLEMENTATION, BUT PLC DOES NOT HANDLE ISOLATED VERTICES WELL + # G = _to_undirected_graph(G) + # unused_node_ids, labels = plc.weakly_connected_components( + # resource_handle=plc.ResourceHandle(), + # graph=G._get_plc_graph(), + # offsets=None, + # indices=None, + # weights=None, + # labels=None, + # do_expensive_check=False, + # ) + # return cp.unique(labels).size + + +@number_connected_components._can_run +def _(G): + # NetworkX <= 3.2.1 does not check directedness for us + try: + return not G.is_directed() + except Exception: + return False + + +@not_implemented_for("directed") +@networkx_algorithm +def connected_components(G): + G = _to_undirected_graph(G) + if G.src_indices.size == 0: + # TODO: PLC doesn't handle empty graphs (or isolated nodes) gracefully! + return [{key} for key in G._nodeiter_to_iter(range(len(G)))] + node_ids, labels = plc.weakly_connected_components( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(), + offsets=None, + indices=None, + weights=None, + labels=None, + do_expensive_check=False, + ) + groups = _groupby(labels, node_ids) + it = (G._nodearray_to_set(connected_ids) for connected_ids in groups.values()) + # TODO: PLC doesn't handle isolated vertices yet, so this is a temporary fix + isolates = _isolates(G) + if isolates.size > 0: + isolates = isolates[isolates > node_ids.max()] + if isolates.size > 0: + it = itertools.chain( + it, ({node} for node in G._nodearray_to_list(isolates)) + ) + return it + + +@not_implemented_for("directed") +@networkx_algorithm +def is_connected(G): + G = _to_undirected_graph(G) + if len(G) == 0: + raise nx.NetworkXPointlessConcept( + "Connectivity is undefined for the null graph." + ) + for community in connected_components(G): + return len(community) == len(G) + raise RuntimeError # pragma: no cover + # PREFERRED IMPLEMENTATION, BUT PLC DOES NOT HANDLE ISOLATED VERTICES WELL + # unused_node_ids, labels = plc.weakly_connected_components( + # resource_handle=plc.ResourceHandle(), + # graph=G._get_plc_graph(), + # offsets=None, + # indices=None, + # weights=None, + # labels=None, + # do_expensive_check=False, + # ) + # return labels.size == len(G) and cp.unique(labels).size == 1 + + +@not_implemented_for("directed") +@networkx_algorithm +def node_connected_component(G, n): + # We could also do plain BFS from n + G = _to_undirected_graph(G) + node_id = n if G.key_to_id is None else G.key_to_id[n] + node_ids, labels = plc.weakly_connected_components( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(), + offsets=None, + indices=None, + weights=None, + labels=None, + do_expensive_check=False, + ) + indices = cp.nonzero(node_ids == node_id)[0] + if indices.size == 0: + return {n} + return G._nodearray_to_set(node_ids[labels == labels[indices[0]]]) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/core.py b/python/nx-cugraph/nx_cugraph/algorithms/core.py new file mode 100644 index 00000000000..2219388bc58 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/core.py @@ -0,0 +1,107 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import pylibcugraph as plc + +import nx_cugraph as nxcg +from nx_cugraph.utils import _get_int_dtype, networkx_algorithm, not_implemented_for + +__all__ = ["k_truss"] + + +@not_implemented_for("directed") +@not_implemented_for("multigraph") +@networkx_algorithm +def k_truss(G, k): + """ + Currently raises `NotImplementedError` for graphs with more than one connected + component when k >= 3. We expect to fix this soon. + """ + if is_nx := isinstance(G, nx.Graph): + G = nxcg.from_networkx(G, preserve_all_attrs=True) + if nxcg.number_of_selfloops(G) > 0: + raise nx.NetworkXError( + "Input graph has self loops which is not permitted; " + "Consider using G.remove_edges_from(nx.selfloop_edges(G))." + ) + + # TODO: create renumbering helper function(s) + if k < 3: + # k-truss graph is comprised of nodes incident on k-2 triangles, so k<3 is a + # boundary condition. Here, all we need to do is drop nodes with zero degree. + # Technically, it would be okay to delete this branch of code, because + # plc.k_truss_subgraph behaves the same for 0 <= k < 3. We keep this branch b/c + # it's faster and has an "early return" if there are no nodes with zero degree. + degrees = G._degrees_array() + # Renumber step 0: node indices + node_indices = degrees.nonzero()[0] + if degrees.size == node_indices.size: + # No change + return G if is_nx else G.copy() + src_indices = G.src_indices + dst_indices = G.dst_indices + # Renumber step 1: edge values (no changes needed) + edge_values = {key: val.copy() for key, val in G.edge_values.items()} + edge_masks = {key: val.copy() for key, val in G.edge_masks.items()} + elif (ncc := nxcg.number_connected_components(G)) > 1: + raise NotImplementedError( + "nx_cugraph.k_truss does not yet work on graphs with more than one " + f"connected component (this graph has {ncc}). We expect to fix this soon." + ) + else: + edge_dtype = _get_int_dtype(G.src_indices.size - 1) + edge_indices = cp.arange(G.src_indices.size, dtype=edge_dtype) + src_indices, dst_indices, edge_indices, _ = plc.k_truss_subgraph( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(edge_array=edge_indices), + k=k, + do_expensive_check=False, + ) + # Renumber step 0: node indices + node_indices = cp.unique(cp.concatenate([src_indices, dst_indices])) + # Renumber step 1: edge values + if edge_indices.dtype != edge_dtype: + # The returned edge_indices may have different dtype (and float) + edge_indices = edge_indices.astype(edge_dtype) + edge_values = {key: val[edge_indices] for key, val in G.edge_values.items()} + edge_masks = {key: val[edge_indices] for key, val in G.edge_masks.items()} + # Renumber step 2: edge indices + mapper = cp.zeros(len(G), src_indices.dtype) + mapper[node_indices] = cp.arange(node_indices.size, dtype=mapper.dtype) + src_indices = mapper[src_indices] + dst_indices = mapper[dst_indices] + # Renumber step 3: node values + node_values = {key: val[node_indices] for key, val in G.node_values.items()} + node_masks = {key: val[node_indices] for key, val in G.node_masks.items()} + # Renumber step 4: key_to_id + if (id_to_key := G.id_to_key) is not None: + key_to_id = { + id_to_key[old_index]: new_index + for new_index, old_index in enumerate(node_indices.tolist()) + } + else: + key_to_id = None + # Same as calling `G.from_coo`, but use __class__ to indicate it's a classmethod. + new_graph = G.__class__.from_coo( + node_indices.size, + src_indices, + dst_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + ) + new_graph.graph.update(G.graph) + return new_graph diff --git a/python/nx-cugraph/nx_cugraph/algorithms/isolate.py b/python/nx-cugraph/nx_cugraph/algorithms/isolate.py index 774627e84f6..d32223fb3ed 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/isolate.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/isolate.py @@ -30,18 +30,18 @@ def is_isolate(G, n): G = _to_graph(G) index = n if G.key_to_id is None else G.key_to_id[n] return not ( - (G.row_indices == index).any().tolist() + (G.src_indices == index).any().tolist() or G.is_directed() - and (G.col_indices == index).any().tolist() + and (G.dst_indices == index).any().tolist() ) def _mark_isolates(G) -> cp.ndarray[bool]: """Return a boolean mask array indicating indices of isolated nodes.""" mark_isolates = cp.ones(len(G), bool) - mark_isolates[G.row_indices] = False + mark_isolates[G.src_indices] = False if G.is_directed(): - mark_isolates[G.col_indices] = False + mark_isolates[G.dst_indices] = False return mark_isolates diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py new file mode 100644 index 00000000000..a68d6940d02 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .hits_alg import * +from .pagerank_alg import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py new file mode 100644 index 00000000000..1c8a47c24b1 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py @@ -0,0 +1,81 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + index_dtype, + networkx_algorithm, +) + +__all__ = ["hits"] + + +@networkx_algorithm( + extra_params={ + 'weight : string or None, optional (default="weight")': ( + "The edge attribute to use as the edge weight." + ), + **_dtype_param, + } +) +def hits( + G, + max_iter=100, + tol=1.0e-8, + nstart=None, + normalized=True, + *, + weight="weight", + dtype=None, +): + G = _to_graph(G, weight, np.float32) + if (N := len(G)) == 0: + return {}, {} + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + nstart = G._dict_to_nodearray(nstart, 0, dtype) + if max_iter <= 0: + if nx.__version__[:3] <= "3.2": + raise ValueError("`maxiter` must be a positive integer.") + raise nx.PowerIterationFailedConvergence(max_iter) + try: + node_ids, hubs, authorities = plc.hits( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(weight, 1, dtype, store_transposed=True), + tol=tol, + initial_hubs_guess_vertices=None + if nstart is None + else cp.arange(N, dtype=index_dtype), + initial_hubs_guess_values=nstart, + max_iter=max_iter, + normalized=normalized, + do_expensive_check=False, + ) + except RuntimeError as exc: + # Errors from PLC are sometimes a little scary and not very helpful + raise nx.PowerIterationFailedConvergence(max_iter) from exc + return ( + G._nodearrays_to_dict(node_ids, hubs), + G._nodearrays_to_dict(node_ids, authorities), + ) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py new file mode 100644 index 00000000000..63f6e89c33a --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py @@ -0,0 +1,112 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + index_dtype, + networkx_algorithm, +) + +__all__ = ["pagerank"] + + +@networkx_algorithm(extra_params=_dtype_param) +def pagerank( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, + *, + dtype=None, +): + """`dangling` parameter is not supported, but it is checked for validity.""" + G = _to_graph(G, weight, 1, np.float32) + if (N := len(G)) == 0: + return {} + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + nstart = G._dict_to_nodearray(nstart, 0, dtype=dtype) + if (total := nstart.sum()) == 0: + raise ZeroDivisionError + nstart /= total + if personalization is not None: + personalization = G._dict_to_nodearray(personalization, 0, dtype=dtype) + if (total := personalization.sum()) == 0: + raise ZeroDivisionError + personalization /= total + if dangling is not None: + # Check if given dangling is valid even though we don't use it + dangling = G._dict_to_nodearray(dangling, 0) # Check validity + if dangling.sum() == 0: + raise ZeroDivisionError + if (G._out_degrees_array() == 0).any(): + raise NotImplementedError("custom dangling weights is not supported") + if max_iter <= 0: + raise nx.PowerIterationFailedConvergence(max_iter) + kwargs = { + "resource_handle": plc.ResourceHandle(), + "graph": G._get_plc_graph(weight, 1, dtype, store_transposed=True), + "precomputed_vertex_out_weight_vertices": None, + "precomputed_vertex_out_weight_sums": None, + "initial_guess_vertices": None + if nstart is None + else cp.arange(N, dtype=index_dtype), + "initial_guess_values": nstart, + "alpha": alpha, + "epsilon": N * tol, + "max_iterations": max_iter, + "do_expensive_check": False, + "fail_on_nonconvergence": False, + } + if personalization is None: + node_ids, values, is_converged = plc.pagerank(**kwargs) + else: + node_ids, values, is_converged = plc.personalized_pagerank( + personalization_vertices=cp.arange(N, dtype=index_dtype), # Why? + personalization_values=personalization, + **kwargs, + ) + if not is_converged: + raise nx.PowerIterationFailedConvergence(max_iter) + return G._nodearrays_to_dict(node_ids, values) + + +@pagerank._can_run +def pagerank( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, + *, + dtype=None, +): + return dangling is None diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/__init__.py new file mode 100644 index 00000000000..b7d6b742176 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .unweighted import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py new file mode 100644 index 00000000000..3413a637b32 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import index_dtype, networkx_algorithm + +__all__ = ["single_source_shortest_path_length", "single_target_shortest_path_length"] + + +@networkx_algorithm +def single_source_shortest_path_length(G, source, cutoff=None): + return _single_shortest_path_length(G, source, cutoff, "Source") + + +@networkx_algorithm +def single_target_shortest_path_length(G, target, cutoff=None): + return _single_shortest_path_length(G, target, cutoff, "Target") + + +def _single_shortest_path_length(G, source, cutoff, kind): + G = _to_graph(G) + if source not in G: + raise nx.NodeNotFound(f"{kind} {source} is not in G") + if G.src_indices.size == 0: + return {source: 0} + if cutoff is None: + cutoff = -1 + src_index = source if G.key_to_id is None else G.key_to_id[source] + distances, predecessors, node_ids = plc.bfs( + handle=plc.ResourceHandle(), + graph=G._get_plc_graph(switch_indices=kind == "Target"), + sources=cp.array([src_index], index_dtype), + direction_optimizing=False, # True for undirected only; what's recommended? + depth_limit=cutoff, + compute_predecessors=False, + do_expensive_check=False, + ) + mask = distances != np.iinfo(distances.dtype).max + return G._nodearrays_to_dict(node_ids[mask], distances[mask]) diff --git a/python/nx-cugraph/nx_cugraph/classes/__init__.py b/python/nx-cugraph/nx_cugraph/classes/__init__.py index e47641ae812..19a5357da55 100644 --- a/python/nx-cugraph/nx_cugraph/classes/__init__.py +++ b/python/nx-cugraph/nx_cugraph/classes/__init__.py @@ -11,5 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from .graph import Graph +from .digraph import DiGraph +from .multigraph import MultiGraph +from .multidigraph import MultiDiGraph -from .digraph import DiGraph # isort:skip +from .function import * diff --git a/python/nx-cugraph/nx_cugraph/classes/digraph.py b/python/nx-cugraph/nx_cugraph/classes/digraph.py index 72a1bff21a9..52ea2334c85 100644 --- a/python/nx-cugraph/nx_cugraph/classes/digraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/digraph.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING +import cupy as cp import networkx as nx import nx_cugraph as nxcg @@ -48,7 +49,7 @@ def number_of_edges( ) -> int: if u is not None or v is not None: raise NotImplementedError - return self.row_indices.size + return self.src_indices.size ########################## # NetworkX graph methods # @@ -59,3 +60,13 @@ def reverse(self, copy: bool = True) -> DiGraph: return self._copy(not copy, self.__class__, reverse=True) # Many more methods to implement... + + ################### + # Private methods # + ################### + + def _in_degrees_array(self): + return cp.bincount(self.dst_indices, minlength=self._N) + + def _out_degrees_array(self): + return cp.bincount(self.src_indices, minlength=self._N) diff --git a/python/nx-cugraph/nx_cugraph/classes/function.py b/python/nx-cugraph/nx_cugraph/classes/function.py new file mode 100644 index 00000000000..633e4abd7f4 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/classes/function.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import networkx_algorithm + +__all__ = ["number_of_selfloops"] + + +@networkx_algorithm +def number_of_selfloops(G): + G = _to_graph(G) + is_selfloop = G.src_indices == G.dst_indices + return is_selfloop.sum().tolist() diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index ded4cc3943f..e32f93d8bfe 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -14,7 +14,7 @@ import operator as op from copy import deepcopy -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING import cupy as cp import networkx as nx @@ -23,8 +23,11 @@ import nx_cugraph as nxcg +from ..utils import index_dtype + if TYPE_CHECKING: # pragma: no cover from collections.abc import Iterable, Iterator + from typing import ClassVar from nx_cugraph.typing import ( AttrKey, @@ -34,6 +37,7 @@ IndexValue, NodeKey, NodeValue, + any_ndarray, ) __all__ = ["Graph"] @@ -43,24 +47,46 @@ class Graph: # Tell networkx to dispatch calls with this object to nx-cugraph - __networkx_plugin__: ClassVar[str] = "cugraph" + __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 + __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 # networkx properties graph: dict graph_attr_dict_factory: ClassVar[type] = dict # Not networkx properties - # We store edge data in COO format with {row,col}_indices and edge_values. - row_indices: cp.ndarray[IndexValue] - col_indices: cp.ndarray[IndexValue] + # We store edge data in COO format with {src,dst}_indices and edge_values. + src_indices: cp.ndarray[IndexValue] + dst_indices: cp.ndarray[IndexValue] edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] edge_masks: dict[AttrKey, cp.ndarray[bool]] - node_values: dict[AttrKey, cp.ndarray[NodeValue]] - node_masks: dict[AttrKey, cp.ndarray[bool]] + node_values: dict[AttrKey, any_ndarray[NodeValue]] + node_masks: dict[AttrKey, any_ndarray[bool]] key_to_id: dict[NodeKey, IndexValue] | None - _id_to_key: dict[IndexValue, NodeKey] | None + _id_to_key: list[NodeKey] | None _N: int + # Used by graph._get_plc_graph + _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { + # signed int + np.dtype(np.int8): np.dtype(np.float32), + np.dtype(np.int16): np.dtype(np.float32), + np.dtype(np.int32): np.dtype(np.float64), + np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 + # unsigned int + np.dtype(np.uint8): np.dtype(np.float32), + np.dtype(np.uint16): np.dtype(np.float32), + np.dtype(np.uint32): np.dtype(np.float64), + np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 + # other + np.dtype(np.bool_): np.dtype(np.float32), + np.dtype(np.float16): np.dtype(np.float32), + } + _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { + np.dtype(np.float32), + np.dtype(np.float64), + } + #################### # Creation methods # #################### @@ -69,32 +95,32 @@ class Graph: def from_coo( cls, N: int, - row_indices: cp.ndarray[IndexValue], - col_indices: cp.ndarray[IndexValue], + src_indices: cp.ndarray[IndexValue], + dst_indices: cp.ndarray[IndexValue], edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, - node_values: dict[AttrKey, cp.ndarray[NodeValue]] | None = None, - node_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, *, key_to_id: dict[NodeKey, IndexValue] | None = None, - id_to_key: dict[IndexValue, NodeKey] | None = None, + id_to_key: list[NodeKey] | None = None, **attr, ) -> Graph: new_graph = object.__new__(cls) - new_graph.row_indices = row_indices - new_graph.col_indices = col_indices + new_graph.src_indices = src_indices + new_graph.dst_indices = dst_indices new_graph.edge_values = {} if edge_values is None else dict(edge_values) new_graph.edge_masks = {} if edge_masks is None else dict(edge_masks) new_graph.node_values = {} if node_values is None else dict(node_values) new_graph.node_masks = {} if node_masks is None else dict(node_masks) new_graph.key_to_id = None if key_to_id is None else dict(key_to_id) - new_graph._id_to_key = None if id_to_key is None else dict(id_to_key) + new_graph._id_to_key = None if id_to_key is None else list(id_to_key) new_graph._N = op.index(N) # Ensure N is integral new_graph.graph = new_graph.graph_attr_dict_factory() new_graph.graph.update(attr) - size = new_graph.row_indices.size + size = new_graph.src_indices.size # Easy and fast sanity checks - if size != new_graph.col_indices.size: + if size != new_graph.dst_indices.size: raise ValueError for attr in ["edge_values", "edge_masks"]: if datadict := getattr(new_graph, attr): @@ -110,31 +136,52 @@ def from_coo( raise ValueError if new_graph._id_to_key is not None and len(new_graph._id_to_key) != N: raise ValueError + if new_graph._id_to_key is not None and new_graph.key_to_id is None: + try: + new_graph.key_to_id = dict(zip(new_graph._id_to_key, range(N))) + except TypeError as exc: + raise ValueError("Bad type of a node value") from exc + if new_graph.src_indices.dtype != index_dtype: + src_indices = new_graph.src_indices.astype(index_dtype) + if not (new_graph.src_indices == src_indices).all(): + raise ValueError( + f"Unable to convert src_indices to {src_indices.dtype.name} " + f"(got {new_graph.src_indices.dtype.name})." + ) + new_graph.src_indices = src_indices + if new_graph.dst_indices.dtype != index_dtype: + dst_indices = new_graph.dst_indices.astype(index_dtype) + if not (new_graph.dst_indices == dst_indices).all(): + raise ValueError( + f"Unable to convert dst_indices to {dst_indices.dtype.name} " + f"(got {new_graph.dst_indices.dtype.name})." + ) + new_graph.dst_indices = dst_indices return new_graph @classmethod def from_csr( cls, indptr: cp.ndarray[IndexValue], - col_indices: cp.ndarray[IndexValue], + dst_indices: cp.ndarray[IndexValue], edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, - node_values: dict[AttrKey, cp.ndarray[NodeValue]] | None = None, - node_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, *, key_to_id: dict[NodeKey, IndexValue] | None = None, - id_to_key: dict[IndexValue, NodeKey] | None = None, + id_to_key: list[NodeKey] | None = None, **attr, ) -> Graph: N = indptr.size - 1 - row_indices = cp.array( + src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead - np.repeat(np.arange(N, dtype=np.int32), cp.diff(indptr).get()) + np.repeat(np.arange(N, dtype=index_dtype), cp.diff(indptr).get()) ) return cls.from_coo( N, - row_indices, - col_indices, + src_indices, + dst_indices, edge_values, edge_masks, node_values, @@ -148,25 +195,25 @@ def from_csr( def from_csc( cls, indptr: cp.ndarray[IndexValue], - row_indices: cp.ndarray[IndexValue], + src_indices: cp.ndarray[IndexValue], edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, - node_values: dict[AttrKey, cp.ndarray[NodeValue]] | None = None, - node_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, *, key_to_id: dict[NodeKey, IndexValue] | None = None, - id_to_key: dict[IndexValue, NodeKey] | None = None, + id_to_key: list[NodeKey] | None = None, **attr, ) -> Graph: N = indptr.size - 1 - col_indices = cp.array( + dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead - np.repeat(np.arange(N, dtype=np.int32), cp.diff(indptr).get()) + np.repeat(np.arange(N, dtype=index_dtype), cp.diff(indptr).get()) ) return cls.from_coo( N, - row_indices, - col_indices, + src_indices, + dst_indices, edge_values, edge_masks, node_values, @@ -180,26 +227,26 @@ def from_csc( def from_dcsr( cls, N: int, - compressed_rows: cp.ndarray[IndexValue], + compressed_srcs: cp.ndarray[IndexValue], indptr: cp.ndarray[IndexValue], - col_indices: cp.ndarray[IndexValue], + dst_indices: cp.ndarray[IndexValue], edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, - node_values: dict[AttrKey, cp.ndarray[NodeValue]] | None = None, - node_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, *, key_to_id: dict[NodeKey, IndexValue] | None = None, - id_to_key: dict[IndexValue, NodeKey] | None = None, + id_to_key: list[NodeKey] | None = None, **attr, ) -> Graph: - row_indices = cp.array( + src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead - np.repeat(compressed_rows.get(), cp.diff(indptr).get()) + np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) ) return cls.from_coo( N, - row_indices, - col_indices, + src_indices, + dst_indices, edge_values, edge_masks, node_values, @@ -213,26 +260,26 @@ def from_dcsr( def from_dcsc( cls, N: int, - compressed_cols: cp.ndarray[IndexValue], + compressed_dsts: cp.ndarray[IndexValue], indptr: cp.ndarray[IndexValue], - row_indices: cp.ndarray[IndexValue], + src_indices: cp.ndarray[IndexValue], edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, - node_values: dict[AttrKey, cp.ndarray[NodeValue]] | None = None, - node_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, *, key_to_id: dict[NodeKey, IndexValue] | None = None, - id_to_key: dict[IndexValue, NodeKey] | None = None, + id_to_key: list[NodeKey] | None = None, **attr, ) -> Graph: - col_indices = cp.array( + dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead - np.repeat(compressed_cols.get(), cp.diff(indptr).get()) + np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) ) return cls.from_coo( N, - row_indices, - col_indices, + src_indices, + dst_indices, edge_values, edge_masks, node_values, @@ -244,7 +291,9 @@ def from_dcsc( def __new__(cls, incoming_graph_data=None, **attr) -> Graph: if incoming_graph_data is None: - new_graph = cls.from_coo(0, cp.empty(0, np.int32), cp.empty(0, np.int32)) + new_graph = cls.from_coo( + 0, cp.empty(0, index_dtype), cp.empty(0, index_dtype) + ) elif incoming_graph_data.__class__ is cls: new_graph = incoming_graph_data.copy() elif incoming_graph_data.__class__ is cls.to_networkx_class(): @@ -295,11 +344,11 @@ def node_dtypes(self) -> dict[AttrKey, Dtype]: return {key: val.dtype for key, val in self.node_values.items()} @property - def id_to_key(self) -> dict[IndexValue, NodeKey] | None: + def id_to_key(self) -> [NodeKey] | None: if self.key_to_id is None: return None if self._id_to_key is None: - self._id_to_key = {val: key for key, val in self.key_to_id.items()} + self._id_to_key = sorted(self.key_to_id, key=self.key_to_id.__getitem__) return self._id_to_key name = nx.Graph.name @@ -335,6 +384,17 @@ def __len__(self) -> int: # NetworkX graph methods # ########################## + @networkx_api + def add_nodes_from(self, nodes_for_adding: Iterable[NodeKey], **attr) -> None: + if self._N != 0: + raise NotImplementedError( + "add_nodes_from is not implemented for graph that already has nodes." + ) + G = self.to_networkx_class()() + G.add_nodes_from(nodes_for_adding, **attr) + G = nxcg.from_networkx(G, preserve_node_attrs=True) + self._become(G) + @networkx_api def clear(self) -> None: self.edge_values.clear() @@ -342,8 +402,8 @@ def clear(self) -> None: self.node_values.clear() self.node_masks.clear() self.graph.clear() - self.row_indices = cp.empty(0, self.row_indices.dtype) - self.col_indices = cp.empty(0, self.col_indices.dtype) + self.src_indices = cp.empty(0, self.src_indices.dtype) + self.dst_indices = cp.empty(0, self.dst_indices.dtype) self._N = 0 self.key_to_id = None self._id_to_key = None @@ -352,8 +412,8 @@ def clear(self) -> None: def clear_edges(self) -> None: self.edge_values.clear() self.edge_masks.clear() - self.row_indices = cp.empty(0, self.row_indices.dtype) - self.col_indices = cp.empty(0, self.col_indices.dtype) + self.src_indices = cp.empty(0, self.src_indices.dtype) + self.dst_indices = cp.empty(0, self.dst_indices.dtype) @networkx_api def copy(self, as_view: bool = False) -> Graph: @@ -370,7 +430,13 @@ def get_edge_data( v = self.key_to_id[v] except KeyError: return default - index = cp.nonzero((self.row_indices == u) & (self.col_indices == v))[0] + else: + try: + if u < 0 or v < 0 or u >= self._N or v >= self._N: + return default + except TypeError: + return default + index = cp.nonzero((self.src_indices == u) & (self.dst_indices == v))[0] if index.size == 0: return default [index] = index.tolist() @@ -390,7 +456,7 @@ def has_edge(self, u: NodeKey, v: NodeKey) -> bool: v = self.key_to_id[v] except KeyError: return False - return bool(((self.row_indices == u) & (self.col_indices == v)).any()) + return bool(((self.src_indices == u) & (self.dst_indices == v)).any()) @networkx_api def has_node(self, n: NodeKey) -> bool: @@ -424,8 +490,8 @@ def order(self) -> int: def size(self, weight: AttrKey | None = None) -> int: if weight is not None: raise NotImplementedError - # If no self-edges, then `self.row_indices.size // 2` - return int((self.row_indices <= self.col_indices).sum()) + # If no self-edges, then `self.src_indices.size // 2` + return int((self.src_indices <= self.dst_indices).sum()) @networkx_api def to_directed(self, as_view: bool = False) -> nxcg.DiGraph: @@ -447,9 +513,9 @@ def to_undirected(self, as_view: bool = False) -> Graph: ################### def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - indptr = self.indptr - row_indices = self.row_indices - col_indices = self.col_indices + # DRY warning: see also MultiGraph._copy + src_indices = self.src_indices + dst_indices = self.dst_indices edge_values = self.edge_values edge_masks = self.edge_masks node_values = self.node_values @@ -457,9 +523,8 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): key_to_id = self.key_to_id id_to_key = None if key_to_id is None else self._id_to_key if not as_view: - indptr = indptr.copy() - row_indices = row_indices.copy() - col_indices = col_indices.copy() + src_indices = src_indices.copy() + dst_indices = dst_indices.copy() edge_values = {key: val.copy() for key, val in edge_values.items()} edge_masks = {key: val.copy() for key, val in edge_masks.items()} node_values = {key: val.copy() for key, val in node_values.items()} @@ -469,11 +534,11 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): if id_to_key is not None: id_to_key = id_to_key.copy() if reverse: - row_indices, col_indices = col_indices, row_indices + src_indices, dst_indices = dst_indices, src_indices rv = cls.from_coo( - indptr, - row_indices, - col_indices, + self._N, + src_indices, + dst_indices, edge_values, edge_masks, node_values, @@ -494,11 +559,16 @@ def _get_plc_graph( edge_dtype: Dtype | None = None, *, store_transposed: bool = False, + switch_indices: bool = False, + edge_array: cp.ndarray[EdgeValue] | None = None, ): - if edge_attr is None: - edge_array = None + if edge_array is not None or edge_attr is None: + pass elif edge_attr not in self.edge_values: - raise KeyError("Graph has no edge attribute {edge_attr!r}") + if edge_default is None: + raise KeyError("Graph has no edge attribute {edge_attr!r}") + # If we were given a default edge value, then it's probably okay to + # use None for the edge_array if we don't have this edge attribute. elif edge_attr not in self.edge_masks: edge_array = self.edge_values[edge_attr] elif not self.edge_masks[edge_attr].all(): @@ -513,25 +583,113 @@ def _get_plc_graph( # Mask is all True; don't need anymore del self.edge_masks[edge_attr] edge_array = self.edge_values[edge_attr] + if edge_array is not None: + if edge_dtype is not None: + edge_dtype = np.dtype(edge_dtype) + if edge_array.dtype != edge_dtype: + edge_array = edge_array.astype(edge_dtype) + # PLC doesn't handle int edge weights right now, so cast int to float + if edge_array.dtype in self._plc_type_map: + if edge_array.dtype == np.int64: + if (val := edge_array.max().tolist()) > 2**53: + raise ValueError( + f"Integer value of value is too large (> 2**53): {val}; " + "pylibcugraph only supports float16 and float32 dtypes." + ) + if (val := edge_array.min().tolist()) < -(2**53): + raise ValueError( + f"Integer value of value is small large (< -2**53): {val}; " + "pylibcugraph only supports float16 and float32 dtypes." + ) + elif ( + edge_array.dtype == np.uint64 + and edge_array.max().tolist() > 2**53 + ): + raise ValueError( + f"Integer value of value is too large (> 2**53): {val}; " + "pylibcugraph only supports float16 and float32 dtypes." + ) + # Consider warning here if we add algorithms that may + # introduce roundoff errors when using floats as ints. + edge_array = edge_array.astype(self._plc_type_map[edge_array.dtype]) + elif edge_array.dtype not in self._plc_allowed_edge_types: + raise TypeError(edge_array.dtype) # Should we cache PLC graph? - if edge_dtype is not None: - edge_dtype = np.dtype(edge_dtype) - if edge_array.dtype != edge_dtype: - edge_array = edge_array.astype(edge_dtype) + src_indices = self.src_indices + dst_indices = self.dst_indices + if switch_indices: + src_indices, dst_indices = dst_indices, src_indices return plc.SGGraph( resource_handle=plc.ResourceHandle(), graph_properties=plc.GraphProperties( is_multigraph=self.is_multigraph(), is_symmetric=not self.is_directed(), ), - src_or_offset_array=self.row_indices, - dst_or_index_array=self.col_indices, + src_or_offset_array=src_indices, + dst_or_index_array=dst_indices, weight_array=edge_array, store_transposed=store_transposed, renumber=False, do_expensive_check=False, ) + def _sort_edge_indices(self, primary="src"): + # DRY warning: see also MultiGraph._sort_edge_indices + if primary == "src": + stacked = cp.vstack((self.dst_indices, self.src_indices)) + elif primary == "dst": + stacked = cp.vstack((self.src_indices, self.dst_indices)) + else: + raise ValueError( + f'Bad `primary` argument; expected "src" or "dst", got {primary!r}' + ) + indices = cp.lexsort(stacked) + if (cp.diff(indices) > 0).all(): + # Already sorted + return + self.src_indices = self.src_indices[indices] + self.dst_indices = self.dst_indices[indices] + self.edge_values.update( + {key: val[indices] for key, val in self.edge_values.items()} + ) + self.edge_masks.update( + {key: val[indices] for key, val in self.edge_masks.items()} + ) + + def _become(self, other: Graph): + if self.__class__ is not other.__class__: + raise TypeError( + "Attempting to update graph inplace with graph of different type!" + ) + self.clear() + edge_values = self.edge_values + edge_masks = self.edge_masks + node_values = self.node_values + node_masks = self.node_masks + graph = self.graph + edge_values.update(other.edge_values) + edge_masks.update(other.edge_masks) + node_values.update(other.node_values) + node_masks.update(other.node_masks) + graph.update(other.graph) + self.__dict__.update(other.__dict__) + self.edge_values = edge_values + self.edge_masks = edge_masks + self.node_values = node_values + self.node_masks = node_masks + self.graph = graph + return self + + def _degrees_array(self): + degrees = cp.bincount(self.src_indices, minlength=self._N) + if self.is_directed(): + degrees += cp.bincount(self.dst_indices, minlength=self._N) + return degrees + + _in_degrees_array = _degrees_array + _out_degrees_array = _degrees_array + + # Data conversions def _nodeiter_to_iter(self, node_ids: Iterable[IndexValue]) -> Iterable[NodeKey]: """Convert an iterable of node IDs to an iterable of node keys.""" if (id_to_key := self.id_to_key) is not None: @@ -543,8 +701,21 @@ def _nodearray_to_list(self, node_ids: cp.ndarray[IndexValue]) -> list[NodeKey]: return node_ids.tolist() return list(self._nodeiter_to_iter(node_ids.tolist())) + def _nodearray_to_set(self, node_ids: cp.ndarray[IndexValue]) -> set[NodeKey]: + if self.key_to_id is None: + return set(node_ids.tolist()) + return set(self._nodeiter_to_iter(node_ids.tolist())) + + def _nodearray_to_dict( + self, values: cp.ndarray[NodeValue] + ) -> dict[NodeKey, NodeValue]: + it = enumerate(values.tolist()) + if (id_to_key := self.id_to_key) is not None: + return {id_to_key[key]: val for key, val in it} + return dict(it) + def _nodearrays_to_dict( - self, node_ids: cp.ndarray[IndexValue], values: cp.ndarray[NodeValue] + self, node_ids: cp.ndarray[IndexValue], values: any_ndarray[NodeValue] ) -> dict[NodeKey, NodeValue]: it = zip(node_ids.tolist(), values.tolist()) if (id_to_key := self.id_to_key) is not None: @@ -574,27 +745,29 @@ def _dict_to_nodearrays( indices_iter = d else: indices_iter = map(self.key_to_id.__getitem__, d) - node_ids = cp.fromiter(indices_iter, np.int32) + node_ids = cp.fromiter(indices_iter, index_dtype) if dtype is None: values = cp.array(list(d.values())) else: values = cp.fromiter(d.values(), dtype) return node_ids, values - # def _dict_to_nodearray( - # self, - # d: dict[NodeKey, NodeValue] | cp.ndarray[NodeValue], - # default: NodeValue | None = None, - # dtype: Dtype | None = None, - # ) -> cp.ndarray[NodeValue]: - # if isinstance(d, cp.ndarray): - # if d.shape[0] != len(self): - # raise ValueError - # return d - # if default is None: - # val_iter = map(d.__getitem__, self) - # else: - # val_iter = (d.get(node, default) for node in self) - # if dtype is None: - # return cp.array(list(val_iter)) - # return cp.fromiter(val_iter, dtype) + def _dict_to_nodearray( + self, + d: dict[NodeKey, NodeValue] | cp.ndarray[NodeValue], + default: NodeValue | None = None, + dtype: Dtype | None = None, + ) -> cp.ndarray[NodeValue]: + if isinstance(d, cp.ndarray): + if d.shape[0] != len(self): + raise ValueError + if dtype is not None and d.dtype != dtype: + return d.astype(dtype) + return d + if default is None: + val_iter = map(d.__getitem__, self) + else: + val_iter = (d.get(node, default) for node in self) + if dtype is None: + return cp.array(list(val_iter)) + return cp.fromiter(val_iter, dtype) diff --git a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py new file mode 100644 index 00000000000..2c7bfc00752 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import networkx as nx + +import nx_cugraph as nxcg + +from .digraph import DiGraph +from .multigraph import MultiGraph + +__all__ = ["MultiDiGraph"] + +networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiDiGraph) + + +class MultiDiGraph(MultiGraph, DiGraph): + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return True + + @classmethod + def to_networkx_class(cls) -> type[nx.MultiDiGraph]: + return nx.MultiDiGraph diff --git a/python/nx-cugraph/nx_cugraph/classes/multigraph.py b/python/nx-cugraph/nx_cugraph/classes/multigraph.py new file mode 100644 index 00000000000..23466dc7dd4 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/classes/multigraph.py @@ -0,0 +1,489 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from copy import deepcopy +from typing import TYPE_CHECKING, ClassVar + +import cupy as cp +import networkx as nx +import numpy as np + +import nx_cugraph as nxcg + +from ..utils import index_dtype +from .graph import Graph + +if TYPE_CHECKING: + from nx_cugraph.typing import ( + AttrKey, + EdgeKey, + EdgeValue, + IndexValue, + NodeKey, + NodeValue, + any_ndarray, + ) +__all__ = ["MultiGraph"] + +networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiGraph) + + +class MultiGraph(Graph): + # networkx properties + edge_key_dict_factory: ClassVar[type] = dict + + # Not networkx properties + + # In a MultiGraph, each edge has a unique `(src, dst, key)` key. + # By default, `key` is 0 if possible, else 1, else 2, etc. + # This key can be any hashable Python object in NetworkX. + # We don't use a dict for our data structure here, because + # that would require a `(src, dst, key)` key. + # Instead, we keep `edge_keys` and/or `edge_indices`. + # `edge_keys` is the list of Python objects for each edge. + # `edge_indices` is for the common case of default multiedge keys, + # in which case we can store it as a cupy array. + # `edge_indices` is generally preferred. It is possible to provide + # both where edge_indices is the default and edge_keys is anything. + # It is also possible for them both to be None, which means the + # default edge indices has not yet been calculated. + edge_indices: cp.ndarray[IndexValue] | None + edge_keys: list[EdgeKey] | None + + #################### + # Creation methods # + #################### + + @classmethod + def from_coo( + cls, + N: int, + src_indices: cp.ndarray[IndexValue], + dst_indices: cp.ndarray[IndexValue], + edge_indices: cp.ndarray[IndexValue] | None = None, + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, + edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, + *, + key_to_id: dict[NodeKey, IndexValue] | None = None, + id_to_key: list[NodeKey] | None = None, + edge_keys: list[EdgeKey] | None = None, + **attr, + ) -> MultiGraph: + new_graph = super().from_coo( + N, + src_indices, + dst_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + id_to_key=id_to_key, + **attr, + ) + new_graph.edge_indices = edge_indices + new_graph.edge_keys = edge_keys + # Easy and fast sanity checks + if ( + new_graph.edge_keys is not None + and len(new_graph.edge_keys) != src_indices.size + ): + raise ValueError + return new_graph + + @classmethod + def from_csr( + cls, + indptr: cp.ndarray[IndexValue], + dst_indices: cp.ndarray[IndexValue], + edge_indices: cp.ndarray[IndexValue] | None = None, + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, + edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, + *, + key_to_id: dict[NodeKey, IndexValue] | None = None, + id_to_key: list[NodeKey] | None = None, + edge_keys: list[EdgeKey] | None = None, + **attr, + ) -> MultiGraph: + N = indptr.size - 1 + src_indices = cp.array( + # cp.repeat is slow to use here, so use numpy instead + np.repeat(np.arange(N, dtype=index_dtype), cp.diff(indptr).get()) + ) + return cls.from_coo( + N, + src_indices, + dst_indices, + edge_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + id_to_key=id_to_key, + edge_keys=edge_keys, + **attr, + ) + + @classmethod + def from_csc( + cls, + indptr: cp.ndarray[IndexValue], + src_indices: cp.ndarray[IndexValue], + edge_indices: cp.ndarray[IndexValue] | None = None, + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, + edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, + *, + key_to_id: dict[NodeKey, IndexValue] | None = None, + id_to_key: list[NodeKey] | None = None, + edge_keys: list[EdgeKey] | None = None, + **attr, + ) -> MultiGraph: + N = indptr.size - 1 + dst_indices = cp.array( + # cp.repeat is slow to use here, so use numpy instead + np.repeat(np.arange(N, dtype=index_dtype), cp.diff(indptr).get()) + ) + return cls.from_coo( + N, + src_indices, + dst_indices, + edge_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + id_to_key=id_to_key, + edge_keys=edge_keys, + **attr, + ) + + @classmethod + def from_dcsr( + cls, + N: int, + compressed_srcs: cp.ndarray[IndexValue], + indptr: cp.ndarray[IndexValue], + dst_indices: cp.ndarray[IndexValue], + edge_indices: cp.ndarray[IndexValue] | None = None, + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, + edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, + *, + key_to_id: dict[NodeKey, IndexValue] | None = None, + id_to_key: list[NodeKey] | None = None, + edge_keys: list[EdgeKey] | None = None, + **attr, + ) -> MultiGraph: + src_indices = cp.array( + # cp.repeat is slow to use here, so use numpy instead + np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) + ) + return cls.from_coo( + N, + src_indices, + dst_indices, + edge_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + id_to_key=id_to_key, + edge_keys=edge_keys, + **attr, + ) + + @classmethod + def from_dcsc( + cls, + N: int, + compressed_dsts: cp.ndarray[IndexValue], + indptr: cp.ndarray[IndexValue], + src_indices: cp.ndarray[IndexValue], + edge_indices: cp.ndarray[IndexValue] | None = None, + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] | None = None, + edge_masks: dict[AttrKey, cp.ndarray[bool]] | None = None, + node_values: dict[AttrKey, any_ndarray[NodeValue]] | None = None, + node_masks: dict[AttrKey, any_ndarray[bool]] | None = None, + *, + key_to_id: dict[NodeKey, IndexValue] | None = None, + id_to_key: list[NodeKey] | None = None, + edge_keys: list[EdgeKey] | None = None, + **attr, + ) -> Graph: + dst_indices = cp.array( + # cp.repeat is slow to use here, so use numpy instead + np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) + ) + return cls.from_coo( + N, + src_indices, + dst_indices, + edge_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + id_to_key=id_to_key, + edge_keys=edge_keys, + **attr, + ) + + def __new__( + cls, incoming_graph_data=None, multigraph_input=None, **attr + ) -> MultiGraph: + if isinstance(incoming_graph_data, dict) and multigraph_input is not False: + new_graph = nxcg.from_networkx( + nx.MultiGraph(incoming_graph_data, multigraph_input=multigraph_input), + preserve_all_attrs=True, + ) + else: + new_graph = super().__new__(cls, incoming_graph_data) + new_graph.graph.update(attr) + return new_graph + + ################# + # Class methods # + ################# + + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False + + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: + return nxcg.MultiDiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.MultiGraph]: + return nx.MultiGraph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[MultiGraph]: + return MultiGraph + + ########################## + # NetworkX graph methods # + ########################## + + @networkx_api + def clear(self) -> None: + super().clear() + self.edge_indices = None + self.edge_keys = None + + @networkx_api + def clear_edges(self) -> None: + super().clear_edges() + self.edge_indices = None + self.edge_keys = None + + @networkx_api + def copy(self, as_view: bool = False) -> MultiGraph: + # Does shallow copy in networkx + return self._copy(as_view, self.__class__) + + @networkx_api + def get_edge_data( + self, + u: NodeKey, + v: NodeKey, + key: EdgeKey | None = None, + default: EdgeValue | None = None, + ): + if self.key_to_id is not None: + try: + u = self.key_to_id[u] + v = self.key_to_id[v] + except KeyError: + return default + else: + try: + if u < 0 or v < 0 or u >= self._N or v >= self._N: + return default + except TypeError: + return default + mask = (self.src_indices == u) & (self.dst_indices == v) + if not mask.any(): + return default + if self.edge_keys is None: + if self.edge_indices is None: + self._calculate_edge_indices() + if key is not None: + try: + mask = mask & (self.edge_indices == key) + except TypeError: + return default + indices = cp.nonzero(mask)[0] + if indices.size == 0: + return default + edge_keys = self.edge_keys + if key is not None and edge_keys is not None: + mask[[i for i in indices.tolist() if edge_keys[i] != key]] = False + indices = cp.nonzero(mask)[0] + if indices.size == 0: + return default + if key is not None: + [index] = indices.tolist() + return { + k: v[index].tolist() + for k, v in self.edge_values.items() + if k not in self.edge_masks or self.edge_masks[k][index] + } + return { + edge_keys[index] + if edge_keys is not None + else index: { + k: v[index].tolist() + for k, v in self.edge_values.items() + if k not in self.edge_masks or self.edge_masks[k][index] + } + for index in indices.tolist() + } + + @networkx_api + def has_edge(self, u: NodeKey, v: NodeKey, key: EdgeKey | None = None) -> bool: + if self.key_to_id is not None: + try: + u = self.key_to_id[u] + v = self.key_to_id[v] + except KeyError: + return False + mask = (self.src_indices == u) & (self.dst_indices == v) + if key is None or (self.edge_indices is None and self.edge_keys is None): + return bool(mask.any()) + if self.edge_keys is None: + try: + return bool((mask & (self.edge_indices == key)).any()) + except TypeError: + return False + indices = cp.nonzero(mask)[0] + if indices.size == 0: + return False + edge_keys = self.edge_keys + return any(edge_keys[i] == key for i in indices.tolist()) + + @networkx_api + def to_directed(self, as_view: bool = False) -> nxcg.MultiDiGraph: + return self._copy(as_view, self.to_directed_class()) + + @networkx_api + def to_undirected(self, as_view: bool = False) -> MultiGraph: + # Does deep copy in networkx + return self.copy(as_view) + + ################### + # Private methods # + ################### + + def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): + # DRY warning: see also Graph._copy + src_indices = self.src_indices + dst_indices = self.dst_indices + edge_indices = self.edge_indices + edge_values = self.edge_values + edge_masks = self.edge_masks + node_values = self.node_values + node_masks = self.node_masks + key_to_id = self.key_to_id + id_to_key = None if key_to_id is None else self._id_to_key + edge_keys = self.edge_keys + if not as_view: + src_indices = src_indices.copy() + dst_indices = dst_indices.copy() + edge_indices = edge_indices.copy() + edge_values = {key: val.copy() for key, val in edge_values.items()} + edge_masks = {key: val.copy() for key, val in edge_masks.items()} + node_values = {key: val.copy() for key, val in node_values.items()} + node_masks = {key: val.copy() for key, val in node_masks.items()} + if key_to_id is not None: + key_to_id = key_to_id.copy() + if id_to_key is not None: + id_to_key = id_to_key.copy() + if edge_keys is not None: + edge_keys = edge_keys.copy() + if reverse: + src_indices, dst_indices = dst_indices, src_indices + rv = cls.from_coo( + self._N, + src_indices, + dst_indices, + edge_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + id_to_key=id_to_key, + edge_keys=edge_keys, + ) + if as_view: + rv.graph = self.graph + else: + rv.graph.update(deepcopy(self.graph)) + return rv + + def _sort_edge_indices(self, primary="src"): + # DRY warning: see also Graph._sort_edge_indices + if self.edge_indices is None and self.edge_keys is None: + return super()._sort_edge_indices(primary=primary) + if primary == "src": + if self.edge_indices is None: + stacked = (self.dst_indices, self.src_indices) + else: + stacked = (self.edge_indices, self.dst_indices, self.src_indices) + elif primary == "dst": + if self.edge_indices is None: + stacked = (self.src_indices, self.dst_indices) + else: + stacked = (self.edge_indices, self.dst_indices, self.src_indices) + else: + raise ValueError( + f'Bad `primary` argument; expected "src" or "dst", got {primary!r}' + ) + indices = cp.lexsort(cp.vstack(stacked)) + if (cp.diff(indices) > 0).all(): + # Already sorted + return + self.src_indices = self.src_indices[indices] + self.dst_indices = self.dst_indices[indices] + self.edge_values.update( + {key: val[indices] for key, val in self.edge_values.items()} + ) + self.edge_masks.update( + {key: val[indices] for key, val in self.edge_masks.items()} + ) + if self.edge_indices is not None: + self.edge_indices = self.edge_indices[indices] + if self.edge_keys is not None: + edge_keys = self.edge_keys + self.edge_keys = [edge_keys[i] for i in indices.tolist()] diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index 1240ea71db7..3c0814370d3 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -24,8 +24,10 @@ import nx_cugraph as nxcg +from .utils import index_dtype + if TYPE_CHECKING: # pragma: no cover - from nx_cugraph.typing import AttrKey, Dtype, EdgeValue, NodeValue + from nx_cugraph.typing import AttrKey, Dtype, EdgeValue, NodeValue, any_ndarray __all__ = [ "from_networkx", @@ -122,8 +124,6 @@ def from_networkx( graph = G else: raise TypeError(f"Expected networkx.Graph; got {type(graph)}") - elif graph.is_multigraph(): - raise NotImplementedError("MultiGraph support is not yet implemented") if preserve_all_attrs: preserve_edge_attrs = True @@ -144,7 +144,7 @@ def from_networkx( else: node_attrs = {node_attrs: None} - if graph.__class__ in {nx.Graph, nx.DiGraph}: + if graph.__class__ in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: # This is a NetworkX private attribute, but is much faster to use adj = graph._adj else: @@ -163,7 +163,11 @@ def from_networkx( edge_attrs = None elif preserve_edge_attrs: # Using comprehensions should be just as fast starting in Python 3.11 - attr_sets = set(map(frozenset, concat(map(dict.values, adj.values())))) + it = concat(map(dict.values, adj.values())) + if graph.is_multigraph(): + it = concat(map(dict.values, it)) + # PERF: should we add `filter(None, ...)` to remove empty data dicts? + attr_sets = set(map(frozenset, it)) attrs = frozenset.union(*attr_sets) edge_attrs = dict.fromkeys(attrs, REQUIRED) if len(attr_sets) > 1: @@ -180,11 +184,19 @@ def from_networkx( if len(required) == 1: # Fast path for the common case of a single attribute with no default [attr] = required - it = ( - attr in edgedata - for rowdata in adj.values() - for edgedata in rowdata.values() - ) + if graph.is_multigraph(): + it = ( + attr in edgedata + for rowdata in adj.values() + for multiedges in rowdata.values() + for edgedata in multiedges.values() + ) + else: + it = ( + attr in edgedata + for rowdata in adj.values() + for edgedata in rowdata.values() + ) if next(it): if all(it): # All edges have data @@ -195,9 +207,10 @@ def from_networkx( del edge_attrs[attr] # Else some edges have attribute (default already None) else: - attr_sets = set( - map(required.intersection, concat(map(dict.values, adj.values()))) - ) + it = concat(map(dict.values, adj.values())) + if graph.is_multigraph(): + it = concat(map(dict.values, it)) + attr_sets = set(map(required.intersection, it)) for attr in required - frozenset.union(*attr_sets): # No edges have these attributes del edge_attrs[attr] @@ -245,7 +258,7 @@ def from_networkx( node_attrs[attr] = REQUIRED key_to_id = dict(zip(adj, range(N))) - col_iter = concat(adj.values()) + dst_iter = concat(adj.values()) try: no_renumber = all(k == v for k, v in key_to_id.items()) except Exception: @@ -253,8 +266,24 @@ def from_networkx( if no_renumber: key_to_id = None else: - col_iter = map(key_to_id.__getitem__, col_iter) - col_indices = cp.fromiter(col_iter, np.int32) + dst_iter = map(key_to_id.__getitem__, dst_iter) + if graph.is_multigraph(): + dst_indices = np.fromiter(dst_iter, index_dtype) + num_multiedges = np.fromiter( + map(len, concat(map(dict.values, adj.values()))), index_dtype + ) + # cp.repeat is slow to use here, so use numpy instead + dst_indices = cp.array(np.repeat(dst_indices, num_multiedges)) + # Determine edge keys and edge ids for multigraphs + edge_keys = list(concat(concat(map(dict.values, adj.values())))) + edge_indices = cp.fromiter( + concat(map(range, map(len, concat(map(dict.values, adj.values()))))), + index_dtype, + ) + if edge_keys == edge_indices.tolist(): + edge_keys = None # Prefer edge_indices + else: + dst_indices = cp.fromiter(dst_iter, index_dtype) edge_values = {} edge_masks = {} @@ -268,16 +297,29 @@ def from_networkx( if edge_default is None: vals = [] append = vals.append - iter_mask = ( - append( - edgedata[edge_attr] - if (present := edge_attr in edgedata) - else False + if graph.is_multigraph(): + iter_mask = ( + append( + edgedata[edge_attr] + if (present := edge_attr in edgedata) + else False + ) + or present + for rowdata in adj.values() + for multiedges in rowdata.values() + for edgedata in multiedges.values() + ) + else: + iter_mask = ( + append( + edgedata[edge_attr] + if (present := edge_attr in edgedata) + else False + ) + or present + for rowdata in adj.values() + for edgedata in rowdata.values() ) - or present - for rowdata in adj.values() - for edgedata in rowdata.values() - ) edge_masks[edge_attr] = cp.fromiter(iter_mask, bool) edge_values[edge_attr] = cp.array(vals, dtype) # if vals.ndim > 1: ... @@ -289,8 +331,16 @@ def from_networkx( # for rowdata in adj.values() # for edgedata in rowdata.values() # ) - iter_values = map( - op.itemgetter(edge_attr), concat(map(dict.values, adj.values())) + it = concat(map(dict.values, adj.values())) + if graph.is_multigraph(): + it = concat(map(dict.values, it)) + iter_values = map(op.itemgetter(edge_attr), it) + elif graph.is_multigraph(): + iter_values = ( + edgedata.get(edge_attr, edge_default) + for rowdata in adj.values() + for multiedges in rowdata.values() + for edgedata in multiedges.values() ) else: iter_values = ( @@ -304,13 +354,14 @@ def from_networkx( edge_values[edge_attr] = cp.fromiter(iter_values, dtype) # if vals.ndim > 1: ... - row_indices = cp.array( - # cp.repeat is slow to use here, so use numpy instead - np.repeat( - np.arange(N, dtype=np.int32), - np.fromiter(map(len, adj.values()), np.int32), - ) + # cp.repeat is slow to use here, so use numpy instead + src_indices = np.repeat( + np.arange(N, dtype=index_dtype), + np.fromiter(map(len, adj.values()), index_dtype), ) + if graph.is_multigraph(): + src_indices = np.repeat(src_indices, num_multiedges) + src_indices = cp.array(src_indices) node_values = {} node_masks = {} @@ -335,8 +386,18 @@ def from_networkx( or present for node_id in adj ) - node_masks[node_attr] = cp.fromiter(iter_mask, bool) - node_values[node_attr] = cp.array(vals, dtype) + # Node values may be numpy or cupy arrays (useful for str, object, etc). + # Someday we'll let the user choose np or cp, and support edge values. + node_mask = np.fromiter(iter_mask, bool) + node_value = np.array(vals, dtype) + try: + node_value = cp.array(node_value) + except ValueError: + pass + else: + node_mask = cp.array(node_mask) + node_values[node_attr] = node_value + node_masks[node_attr] = node_mask # if vals.ndim > 1: ... else: if node_default is REQUIRED: @@ -345,34 +406,58 @@ def from_networkx( iter_values = ( nodes[node_id].get(node_attr, node_default) for node_id in adj ) + # Node values may be numpy or cupy arrays (useful for str, object, etc). + # Someday we'll let the user choose np or cp, and support edge values. if dtype is None: - node_values[node_attr] = cp.array(list(iter_values)) + node_value = np.array(list(iter_values)) else: - node_values[node_attr] = cp.fromiter(iter_values, dtype) + node_value = np.fromiter(iter_values, dtype) + try: + node_value = cp.array(node_value) + except ValueError: + pass + node_values[node_attr] = node_value # if vals.ndim > 1: ... - - if graph.is_directed() or as_directed: - klass = nxcg.DiGraph + if graph.is_multigraph(): + if graph.is_directed() or as_directed: + klass = nxcg.MultiDiGraph + else: + klass = nxcg.MultiGraph + rv = klass.from_coo( + N, + src_indices, + dst_indices, + edge_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + edge_keys=edge_keys, + ) else: - klass = nxcg.Graph - rv = klass.from_coo( - N, - row_indices, - col_indices, - edge_values, - edge_masks, - node_values, - node_masks, - key_to_id=key_to_id, - ) + if graph.is_directed() or as_directed: + klass = nxcg.DiGraph + else: + klass = nxcg.Graph + rv = klass.from_coo( + N, + src_indices, + dst_indices, + edge_values, + edge_masks, + node_values, + node_masks, + key_to_id=key_to_id, + ) if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? return rv def _iter_attr_dicts( - values: dict[AttrKey, cp.ndarray[EdgeValue | NodeValue]], - masks: dict[AttrKey, cp.ndarray[bool]], + values: dict[AttrKey, any_ndarray[EdgeValue | NodeValue]], + masks: dict[AttrKey, any_ndarray[bool]], ): full_attrs = list(values.keys() - masks.keys()) if full_attrs: @@ -398,7 +483,7 @@ def _iter_attr_dicts( return full_dicts -def to_networkx(G: nxcg.Graph) -> nx.Graph: +def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: """Convert a nx_cugraph graph to networkx graph. All edge and node attributes and ``G.graph`` properties are converted. @@ -406,6 +491,11 @@ def to_networkx(G: nxcg.Graph) -> nx.Graph: Parameters ---------- G : nx_cugraph.Graph + sort_edges : bool, default False + Whether to sort the edge data of the input graph by (src, dst) indices + before converting. This can be useful to convert to networkx graphs + that iterate over edges consistently since edges are stored in dicts + in the order they were added. Returns ------- @@ -417,6 +507,8 @@ def to_networkx(G: nxcg.Graph) -> nx.Graph: """ rv = G.to_networkx_class()() id_to_key = G.id_to_key + if sort_edges: + G._sort_edge_indices() node_values = G.node_values node_masks = G.node_masks @@ -427,32 +519,43 @@ def to_networkx(G: nxcg.Graph) -> nx.Graph: full_node_dicts = _iter_attr_dicts(node_values, node_masks) rv.add_nodes_from(zip(node_iter, full_node_dicts)) elif id_to_key is not None: - rv.add_nodes_from(id_to_key.values()) + rv.add_nodes_from(id_to_key) else: rv.add_nodes_from(range(len(G))) - row_indices = G.row_indices - col_indices = G.col_indices + src_indices = G.src_indices + dst_indices = G.dst_indices edge_values = G.edge_values edge_masks = G.edge_masks - if edge_values and not G.is_directed(): + if not G.is_directed(): # Only add upper triangle of the adjacency matrix so we don't double-add edges - mask = row_indices <= col_indices - row_indices = row_indices[mask] - col_indices = col_indices[mask] - edge_values = {k: v[mask] for k, v in edge_values.items()} + mask = src_indices <= dst_indices + src_indices = src_indices[mask] + dst_indices = dst_indices[mask] + if edge_values: + edge_values = {k: v[mask] for k, v in edge_values.items()} if edge_masks: edge_masks = {k: v[mask] for k, v in edge_masks.items()} - row_indices = row_iter = row_indices.tolist() - col_indices = col_iter = col_indices.tolist() + src_indices = src_iter = src_indices.tolist() + dst_indices = dst_iter = dst_indices.tolist() if id_to_key is not None: - row_iter = map(id_to_key.__getitem__, row_indices) - col_iter = map(id_to_key.__getitem__, col_indices) - if edge_values: + src_iter = map(id_to_key.__getitem__, src_indices) + dst_iter = map(id_to_key.__getitem__, dst_indices) + if G.is_multigraph() and (G.edge_keys is not None or G.edge_indices is not None): + if G.edge_keys is not None: + edge_keys = G.edge_keys + else: + edge_keys = G.edge_indices.tolist() + if edge_values: + full_edge_dicts = _iter_attr_dicts(edge_values, edge_masks) + rv.add_edges_from(zip(src_iter, dst_iter, edge_keys, full_edge_dicts)) + else: + rv.add_edges_from(zip(src_iter, dst_iter, edge_keys)) + elif edge_values: full_edge_dicts = _iter_attr_dicts(edge_values, edge_masks) - rv.add_edges_from(zip(row_iter, col_iter, full_edge_dicts)) + rv.add_edges_from(zip(src_iter, dst_iter, full_edge_dicts)) else: - rv.add_edges_from(zip(row_iter, col_iter)) + rv.add_edges_from(zip(src_iter, dst_iter)) rv.graph.update(G.graph) return rv diff --git a/python/nx-cugraph/nx_cugraph/convert_matrix.py b/python/nx-cugraph/nx_cugraph/convert_matrix.py new file mode 100644 index 00000000000..6c8b8fb4a1d --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/convert_matrix.py @@ -0,0 +1,146 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import numpy as np + +from .generators._utils import _create_using_class +from .utils import index_dtype, networkx_algorithm + +__all__ = [ + "from_pandas_edgelist", + "from_scipy_sparse_array", +] + + +@networkx_algorithm +def from_pandas_edgelist( + df, + source="source", + target="target", + edge_attr=None, + create_using=None, + edge_key=None, +): + """cudf.DataFrame inputs also supported.""" + graph_class, inplace = _create_using_class(create_using) + src_array = df[source].to_numpy() + dst_array = df[target].to_numpy() + # Renumber step 0: node keys + nodes = np.unique(np.concatenate([src_array, dst_array])) + N = nodes.size + kwargs = {} + if N > 0 and ( + nodes[0] != 0 + or nodes[N - 1] != N - 1 + or ( + nodes.dtype.kind not in {"i", "u"} + and not (nodes == np.arange(N, dtype=np.int64)).all() + ) + ): + # We need to renumber indices--np.searchsorted to the rescue! + kwargs["id_to_key"] = nodes.tolist() + src_indices = cp.array(np.searchsorted(nodes, src_array), index_dtype) + dst_indices = cp.array(np.searchsorted(nodes, dst_array), index_dtype) + else: + src_indices = cp.array(src_array) + dst_indices = cp.array(dst_array) + + if not graph_class.is_directed(): + # Symmetrize the edges + mask = src_indices != dst_indices + if mask.all(): + mask = None + src_indices, dst_indices = ( + cp.hstack( + (src_indices, dst_indices[mask] if mask is not None else dst_indices) + ), + cp.hstack( + (dst_indices, src_indices[mask] if mask is not None else src_indices) + ), + ) + + if edge_attr is not None: + # Additional columns requested for edge data + if edge_attr is True: + attr_col_headings = df.columns.difference({source, target}).to_list() + elif isinstance(edge_attr, (list, tuple)): + attr_col_headings = edge_attr + else: + attr_col_headings = [edge_attr] + if len(attr_col_headings) == 0: + raise nx.NetworkXError( + "Invalid edge_attr argument: No columns found with name: " + f"{attr_col_headings}" + ) + try: + edge_values = { + key: cp.array(val.to_numpy()) + for key, val in df[attr_col_headings].items() + } + except (KeyError, TypeError) as exc: + raise nx.NetworkXError(f"Invalid edge_attr argument: {edge_attr}") from exc + + if not graph_class.is_directed(): + # Symmetrize the edges + edge_values = { + key: cp.hstack((val, val[mask] if mask is not None else val)) + for key, val in edge_values.items() + } + kwargs["edge_values"] = edge_values + + if graph_class.is_multigraph() and edge_key is not None: + try: + edge_keys = df[edge_key].to_list() + except (KeyError, TypeError) as exc: + raise nx.NetworkXError( + f"Invalid edge_key argument: {edge_key}" + ) from exc + if not graph_class.is_directed(): + # Symmetrize the edges + edge_keys = cp.hstack( + (edge_keys, edge_keys[mask] if mask is not None else edge_keys) + ) + kwargs["edge_keys"] = edge_keys + + G = graph_class.from_coo(N, src_indices, dst_indices, **kwargs) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def from_scipy_sparse_array( + A, parallel_edges=False, create_using=None, edge_attribute="weight" +): + graph_class, inplace = _create_using_class(create_using) + m, n = A.shape + if m != n: + raise nx.NetworkXError(f"Adjacency matrix not square: nx,ny={A.shape}") + if A.format != "coo": + A = A.tocoo() + if A.dtype.kind in {"i", "u"} and graph_class.is_multigraph() and parallel_edges: + src_indices = cp.array(np.repeat(A.row, A.data), index_dtype) + dst_indices = cp.array(np.repeat(A.col, A.data), index_dtype) + weight = cp.empty(src_indices.size, A.data.dtype) + weight[:] = 1 + else: + src_indices = cp.array(A.row, index_dtype) + dst_indices = cp.array(A.col, index_dtype) + weight = cp.array(A.data) + G = graph_class.from_coo( + n, src_indices, dst_indices, edge_values={"weight": weight} + ) + if inplace: + return create_using._become(G) + return G diff --git a/python/nx-cugraph/nx_cugraph/generators/__init__.py b/python/nx-cugraph/nx_cugraph/generators/__init__.py new file mode 100644 index 00000000000..c1834a4dec7 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/generators/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .classic import * +from .community import * +from .small import * +from .social import * diff --git a/python/nx-cugraph/nx_cugraph/generators/_utils.py b/python/nx-cugraph/nx_cugraph/generators/_utils.py new file mode 100644 index 00000000000..e38ace5b28d --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/generators/_utils.py @@ -0,0 +1,136 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import operator as op + +import cupy as cp +import networkx as nx + +import nx_cugraph as nxcg + +from ..utils import index_dtype + +# 3.2.1 fixed some issues in generators that occur in 3.2 and earlier +_IS_NX32_OR_LESS = (nxver := nx.__version__)[:3] <= "3.2" and ( + len(nxver) <= 3 or nxver[3] != "." and not nxver[3].isdigit() +) + + +def _ensure_int(n): + """Ensure n is integral.""" + return op.index(n) + + +def _ensure_nonnegative_int(n): + """Ensure n is a nonnegative integer.""" + n = op.index(n) + if n < 0: + raise nx.NetworkXError(f"Negative number of nodes not valid: {n}") + return n + + +def _complete_graph_indices(n): + all_indices = cp.indices((n, n), dtype=index_dtype) + src_indices = all_indices[0].ravel() + dst_indices = all_indices[1].ravel() + del all_indices + mask = src_indices != dst_indices + return (src_indices[mask], dst_indices[mask]) + + +def _common_small_graph(n, nodes, create_using, *, allow_directed=True): + """Create a "common graph" for small n. + + n == 0: empty graph + n == 1: empty graph + n == 2: complete graph + n > 2: undefined + """ + graph_class, inplace = _create_using_class(create_using) + if not allow_directed and graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + if n < 2: + G = graph_class.from_coo( + n, cp.empty(0, index_dtype), cp.empty(0, index_dtype), id_to_key=nodes + ) + else: + G = graph_class.from_coo( + n, + cp.arange(2, dtype=index_dtype), + cp.array([1, 0], index_dtype), + id_to_key=nodes, + ) + if inplace: + return create_using._become(G) + return G + + +def _create_using_class(create_using, *, default=nxcg.Graph): + """Handle ``create_using`` argument and return a Graph type from nx_cugraph.""" + inplace = False + if create_using is None: + G = default() + elif isinstance(create_using, type): + G = create_using() + elif not hasattr(create_using, "is_directed") or not hasattr( + create_using, "is_multigraph" + ): + raise TypeError("create_using is not a valid graph type or instance") + elif not isinstance(create_using, nxcg.Graph): + raise NotImplementedError( + f"create_using with object of type {type(create_using)} is not supported " + "by the cugraph backend; only nx_cugraph.Graph objects are allowed." + ) + else: + inplace = True + G = create_using + G.clear() + if not isinstance(G, nxcg.Graph): + if G.is_multigraph(): + if G.is_directed(): + graph_class = nxcg.MultiDiGraph + else: + graph_class = nxcg.MultiGraph + elif G.is_directed(): + graph_class = nxcg.DiGraph + else: + graph_class = nxcg.Graph + if G.__class__ not in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: + raise NotImplementedError( + f"create_using with type {type(G)} is not supported by the cugraph " + "backend; only standard networkx or nx_cugraph Graph objects are " + "allowed (but not customized subclasses derived from them)." + ) + else: + graph_class = G.__class__ + return graph_class, inplace + + +def _number_and_nodes(n_and_nodes): + n, nodes = n_and_nodes + try: + n = op.index(n) + except TypeError: + n = len(nodes) + if n < 0: + raise nx.NetworkXError(f"Negative number of nodes not valid: {n}") + if not isinstance(nodes, list): + nodes = list(nodes) + if not nodes: + return (n, None) + if nodes[0] == 0 and nodes[n - 1] == n - 1: + try: + if nodes == list(range(n)): + return (n, None) + except Exception: + pass + return (n, nodes) diff --git a/python/nx-cugraph/nx_cugraph/generators/classic.py b/python/nx-cugraph/nx_cugraph/generators/classic.py new file mode 100644 index 00000000000..b196c232320 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/generators/classic.py @@ -0,0 +1,423 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import itertools +from numbers import Integral + +import cupy as cp +import networkx as nx +import numpy as np + +import nx_cugraph as nxcg + +from ..utils import _get_int_dtype, index_dtype, networkx_algorithm, nodes_or_number +from ._utils import ( + _IS_NX32_OR_LESS, + _common_small_graph, + _complete_graph_indices, + _create_using_class, + _ensure_int, + _ensure_nonnegative_int, + _number_and_nodes, +) + +__all__ = [ + "barbell_graph", + "circular_ladder_graph", + "complete_graph", + "complete_multipartite_graph", + "cycle_graph", + "empty_graph", + "ladder_graph", + "lollipop_graph", + "null_graph", + "path_graph", + "star_graph", + "tadpole_graph", + "trivial_graph", + "turan_graph", + "wheel_graph", +] + +concat = itertools.chain.from_iterable + + +@networkx_algorithm +def barbell_graph(m1, m2, create_using=None): + # Like two complete graphs and a path_graph + m1 = _ensure_nonnegative_int(m1) + if m1 < 2: + raise nx.NetworkXError("Invalid graph description, m1 should be >=2") + m2 = _ensure_nonnegative_int(m2) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_bell1, dst_bell1 = _complete_graph_indices(m1) + src_bell2 = src_bell1 + (m1 + m2) + dst_bell2 = dst_bell1 + (m1 + m2) + if m2 == 0: + src_bar = cp.array([m1 - 1, m1], index_dtype) + dst_bar = cp.array([m1, m1 - 1], index_dtype) + else: + src_bar = cp.arange(2 * m1 - 1, 2 * m1 + 2 * m2 + 1, dtype=index_dtype) // 2 + dst_bar = ( + cp.arange(m1 - 1, m1 + m2 + 1, dtype=index_dtype)[:, None] + + cp.array([-1, 1], index_dtype) + ).ravel()[1:-1] + src_indices = cp.hstack((src_bell1, src_bar, src_bell2)) + dst_indices = cp.hstack((dst_bell1, dst_bar, dst_bell2)) + G = graph_class.from_coo(2 * m1 + m2, src_indices, dst_indices) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def circular_ladder_graph(n, create_using=None): + return _ladder_graph(n, create_using, is_circular=True) + + +@nodes_or_number(0) +@networkx_algorithm +def complete_graph(n, create_using=None): + n, nodes = _number_and_nodes(n) + if n < 3: + return _common_small_graph(n, nodes, create_using) + graph_class, inplace = _create_using_class(create_using) + src_indices, dst_indices = _complete_graph_indices(n) + G = graph_class.from_coo(n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def complete_multipartite_graph(*subset_sizes): + if not subset_sizes: + return nxcg.Graph() + try: + subset_sizes = [_ensure_int(size) for size in subset_sizes] + except TypeError: + subsets = [list(subset) for subset in subset_sizes] + subset_sizes = [len(subset) for subset in subsets] + nodes = list(concat(subsets)) + else: + subsets = nodes = None + try: + subset_sizes = [_ensure_nonnegative_int(size) for size in subset_sizes] + except nx.NetworkXError: + if _IS_NX32_OR_LESS: + raise NotImplementedError("Negative number of nodes is not supported") + raise + L1 = [] + L2 = [] + total = 0 + for size in subset_sizes: + all_indices = cp.indices((total, size), dtype=index_dtype) + L1.append(all_indices[0].ravel()) + L2.append(all_indices[1].ravel() + total) + total += size + src_indices = cp.hstack(L1 + L2) + dst_indices = cp.hstack(L2 + L1) + subsets_array = cp.array( + np.repeat( + np.arange(len(subset_sizes), dtype=_get_int_dtype(len(subset_sizes) - 1)), + subset_sizes, + ) + ) + return nxcg.Graph.from_coo( + subsets_array.size, + src_indices, + dst_indices, + node_values={"subset": subsets_array}, + id_to_key=nodes, + ) + + +@nodes_or_number(0) +@networkx_algorithm +def cycle_graph(n, create_using=None): + n, nodes = _number_and_nodes(n) + graph_class, inplace = _create_using_class(create_using) + if n == 1: + src_indices = cp.zeros(1, index_dtype) + dst_indices = cp.zeros(1, index_dtype) + elif n == 2 and graph_class.is_multigraph() and not graph_class.is_directed(): + # This is kind of a peculiar edge case + src_indices = cp.array([0, 0, 1, 1], index_dtype) + dst_indices = cp.array([1, 1, 0, 0], index_dtype) + elif n < 3: + return _common_small_graph(n, nodes, create_using) + elif graph_class.is_directed(): + src_indices = cp.arange(n, dtype=index_dtype) + dst_indices = cp.arange(1, n + 1, dtype=index_dtype) + dst_indices[-1] = 0 + else: + src_indices = cp.arange(2 * n, dtype=index_dtype) // 2 + dst_indices = ( + cp.arange(n, dtype=index_dtype)[:, None] + cp.array([-1, 1], index_dtype) + ).ravel() + dst_indices[0] = n - 1 + dst_indices[-1] = 0 + G = graph_class.from_coo(n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + + +@nodes_or_number(0) +@networkx_algorithm +def empty_graph(n=0, create_using=None, default=nx.Graph): + n, nodes = _number_and_nodes(n) + graph_class, inplace = _create_using_class(create_using, default=default) + G = graph_class.from_coo( + n, cp.empty(0, index_dtype), cp.empty(0, index_dtype), id_to_key=nodes + ) + if inplace: + return create_using._become(G) + return G + + +def _ladder_graph(n, create_using, *, is_circular=False): + # Like path path_graph with extra arange, and middle link missing + n = _ensure_nonnegative_int(n) + if n < 2: + if not is_circular: + return _common_small_graph(2 * n, None, create_using, allow_directed=False) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + if n == 1: + src_indices = cp.array([0, 1, 0, 1], index_dtype) + dst_indices = cp.array([0, 0, 1, 1], index_dtype) + nodes = None + elif graph_class.is_multigraph(): + src_indices = cp.array([0, 0, 1, 1], index_dtype) + dst_indices = cp.array([1, 1, 0, 0], index_dtype) + nodes = [0, -1] + else: + src_indices = cp.array([0, 1], index_dtype) + dst_indices = cp.array([1, 0], index_dtype) + nodes = [0, -1] + G = graph_class.from_coo(2, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + path_src = cp.arange(1, 2 * n - 1, dtype=index_dtype) // 2 + path_dst = ( + cp.arange(n, dtype=index_dtype)[:, None] + cp.array([-1, 1], index_dtype) + ).ravel()[1:-1] + srcs = [path_src, path_src + n, cp.arange(2 * n, dtype=index_dtype)] + dsts = [ + path_dst, + path_dst + n, + cp.arange(n, 2 * n, dtype=index_dtype), + cp.arange(0, n, dtype=index_dtype), + ] + if is_circular and (n > 2 or graph_class.is_multigraph()): + srcs.append(cp.array([0, n - 1, n, 2 * n - 1], index_dtype)) + dsts.append(cp.array([n - 1, 0, 2 * n - 1, n], index_dtype)) + src_indices = cp.hstack(srcs) + dst_indices = cp.hstack(dsts) + G = graph_class.from_coo(2 * n, src_indices, dst_indices) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def ladder_graph(n, create_using=None): + return _ladder_graph(n, create_using) + + +@nodes_or_number([0, 1]) +@networkx_algorithm +def lollipop_graph(m, n, create_using=None): + # Like complete_graph then path_graph + orig_m, unused_nodes_m = m + orig_n, unused_nodes_n = n + m, m_nodes = _number_and_nodes(m) + if m < 2: + raise nx.NetworkXError( + "Invalid description: m should indicate at least 2 nodes" + ) + n, n_nodes = _number_and_nodes(n) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + msrc_indices, mdst_indices = _complete_graph_indices(m) + nsrc_indices = cp.arange(2 * m - 1, 2 * m + 2 * n - 1, dtype=index_dtype) // 2 + ndst_indices = ( + cp.arange(m - 1, m + n, dtype=index_dtype)[:, None] + + cp.array([-1, 1], index_dtype) + ).ravel()[1:-1] + src_indices = cp.hstack((msrc_indices, nsrc_indices)) + dst_indices = cp.hstack((mdst_indices, ndst_indices)) + if isinstance(orig_m, Integral) and isinstance(orig_n, Integral): + nodes = None + else: + nodes = list(range(m)) if m_nodes is None else m_nodes + nodes.extend(range(n) if n_nodes is None else n_nodes) + if len(set(nodes)) != len(nodes): + raise nx.NetworkXError("Nodes must be distinct in containers m and n") + G = graph_class.from_coo(m + n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def null_graph(create_using=None): + return _common_small_graph(0, None, create_using) + + +@nodes_or_number(0) +@networkx_algorithm +def path_graph(n, create_using=None): + n, nodes = _number_and_nodes(n) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + src_indices = cp.arange(n - 1, dtype=index_dtype) + dst_indices = cp.arange(1, n, dtype=index_dtype) + elif n < 3: + return _common_small_graph(n, nodes, create_using) + else: + src_indices = cp.arange(1, 2 * n - 1, dtype=index_dtype) // 2 + dst_indices = ( + cp.arange(n, dtype=index_dtype)[:, None] + cp.array([-1, 1], index_dtype) + ).ravel()[1:-1] + G = graph_class.from_coo(n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + + +@nodes_or_number(0) +@networkx_algorithm +def star_graph(n, create_using=None): + orig_n, orig_nodes = n + n, nodes = _number_and_nodes(n) + # star_graph behaves differently whether the input was an int or iterable + if isinstance(orig_n, Integral): + if nodes is not None: + nodes.append(n) + n += 1 + if n < 3: + return _common_small_graph(n, nodes, create_using, allow_directed=False) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + flat = cp.zeros(n - 1, index_dtype) + ramp = cp.arange(1, n, dtype=index_dtype) + src_indices = cp.hstack((flat, ramp)) + dst_indices = cp.hstack((ramp, flat)) + G = graph_class.from_coo(n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + + +@nodes_or_number([0, 1]) +@networkx_algorithm +def tadpole_graph(m, n, create_using=None): + orig_m, unused_nodes_m = m + orig_n, unused_nodes_n = n + m, m_nodes = _number_and_nodes(m) + if m < 2: + raise nx.NetworkXError( + "Invalid description: m should indicate at least 2 nodes" + ) + n, n_nodes = _number_and_nodes(n) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + if isinstance(orig_m, Integral) and isinstance(orig_n, Integral): + nodes = None + else: + nodes = list(range(m)) if m_nodes is None else m_nodes + nodes.extend(range(n) if n_nodes is None else n_nodes) + if m == 2 and not graph_class.is_multigraph(): + src_indices = cp.arange(1, 2 * (m + n) - 1, dtype=index_dtype) // 2 + dst_indices = ( + cp.arange((m + n), dtype=index_dtype)[:, None] + + cp.array([-1, 1], index_dtype) + ).ravel()[1:-1] + else: + src_indices = cp.arange(2 * (m + n), dtype=index_dtype) // 2 + dst_indices = ( + cp.arange((m + n), dtype=index_dtype)[:, None] + + cp.array([-1, 1], index_dtype) + ).ravel() + dst_indices[0] = m - 1 + dst_indices[-1] = 0 + G = graph_class.from_coo(m + n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def trivial_graph(create_using=None): + return _common_small_graph(1, None, create_using) + + +@networkx_algorithm +def turan_graph(n, r): + if not 1 <= r <= n: + raise nx.NetworkXError("Must satisfy 1 <= r <= n") + n_div_r, n_mod_r = divmod(n, r) + partitions = [n_div_r] * (r - n_mod_r) + [n_div_r + 1] * n_mod_r + return complete_multipartite_graph(*partitions) + + +@nodes_or_number(0) +@networkx_algorithm +def wheel_graph(n, create_using=None): + n, nodes = _number_and_nodes(n) + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + if n < 2: + G = graph_class.from_coo( + n, cp.empty(0, index_dtype), cp.empty(0, index_dtype), id_to_key=nodes + ) + else: + # Like star_graph + flat = cp.zeros(n - 1, index_dtype) + ramp = cp.arange(1, n, dtype=index_dtype) + # Like cycle_graph + if n < 3: + src_indices = cp.empty(0, index_dtype) + dst_indices = cp.empty(0, index_dtype) + elif n > 3: + src_indices = cp.arange(2, 2 * n, dtype=index_dtype) // 2 + dst_indices = ( + cp.arange(1, n, dtype=index_dtype)[:, None] + + cp.array([-1, 1], index_dtype) + ).ravel() + dst_indices[-1] = 1 + dst_indices[0] = n - 1 + elif graph_class.is_multigraph(): + src_indices = cp.array([1, 1, 2, 2], index_dtype) + dst_indices = cp.array([2, 2, 1, 1], index_dtype) + else: + src_indices = cp.array([1, 2], index_dtype) + dst_indices = cp.array([2, 1], index_dtype) + src_indices = cp.hstack((flat, ramp, src_indices)) + dst_indices = cp.hstack((ramp, flat, dst_indices)) + G = graph_class.from_coo(n, src_indices, dst_indices, id_to_key=nodes) + if inplace: + return create_using._become(G) + return G diff --git a/python/nx-cugraph/nx_cugraph/generators/community.py b/python/nx-cugraph/nx_cugraph/generators/community.py new file mode 100644 index 00000000000..e5cb03e8cc0 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/generators/community.py @@ -0,0 +1,45 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp + +import nx_cugraph as nxcg + +from ..utils import networkx_algorithm +from ._utils import ( + _common_small_graph, + _complete_graph_indices, + _ensure_int, + _ensure_nonnegative_int, +) + +__all__ = [ + "caveman_graph", +] + + +@networkx_algorithm +def caveman_graph(l, k): # noqa: E741 + l = _ensure_int(l) # noqa: E741 + k = _ensure_int(k) + N = _ensure_nonnegative_int(k * l) + if l == 0 or k < 1: + return _common_small_graph(N, None, None) + k = _ensure_nonnegative_int(k) + src_clique, dst_clique = _complete_graph_indices(k) + src_cliques = [src_clique] + dst_cliques = [dst_clique] + src_cliques.extend(src_clique + i * k for i in range(1, l)) + dst_cliques.extend(dst_clique + i * k for i in range(1, l)) + src_indices = cp.hstack(src_cliques) + dst_indices = cp.hstack(dst_cliques) + return nxcg.Graph.from_coo(l * k, src_indices, dst_indices) diff --git a/python/nx-cugraph/nx_cugraph/generators/small.py b/python/nx-cugraph/nx_cugraph/generators/small.py new file mode 100644 index 00000000000..b9a189c31d5 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/generators/small.py @@ -0,0 +1,622 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx + +import nx_cugraph as nxcg + +from ..utils import index_dtype, networkx_algorithm +from ._utils import _IS_NX32_OR_LESS, _create_using_class + +__all__ = [ + "bull_graph", + "chvatal_graph", + "cubical_graph", + "desargues_graph", + "diamond_graph", + "dodecahedral_graph", + "frucht_graph", + "heawood_graph", + "house_graph", + "house_x_graph", + "icosahedral_graph", + "krackhardt_kite_graph", + "moebius_kantor_graph", + "octahedral_graph", + "pappus_graph", + "petersen_graph", + "sedgewick_maze_graph", + "tetrahedral_graph", + "truncated_cube_graph", + "truncated_tetrahedron_graph", + "tutte_graph", +] + + +@networkx_algorithm +def bull_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_indices = cp.array([0, 0, 1, 1, 1, 2, 2, 2, 3, 4], index_dtype) + dst_indices = cp.array([1, 2, 0, 2, 3, 0, 1, 4, 1, 2], index_dtype) + G = graph_class.from_coo(5, src_indices, dst_indices, name="Bull Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def chvatal_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, + 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, + 11, 11, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 4, 6, 9, 0, 2, 5, 7, 1, 3, 6, 8, 2, 4, 7, 9, 0, 3, 5, 8, 1, 4, 10, 11, + 0, 2, 10, 11, 1, 3, 8, 11, 2, 4, 7, 10, 0, 3, 10, 11, 5, 6, 8, 9, 5, 6, + 7, 9, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo(12, src_indices, dst_indices, name="Chvatal Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def cubical_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_indices = cp.array( + [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7], + index_dtype, + ) + dst_indices = cp.array( + [1, 3, 4, 0, 2, 7, 1, 3, 6, 0, 2, 5, 0, 5, 7, 3, 4, 6, 2, 5, 7, 1, 4, 6], + index_dtype, + ) + name = ("Platonic Cubical Graph",) if _IS_NX32_OR_LESS else "Platonic Cubical Graph" + G = graph_class.from_coo(8, src_indices, dst_indices, name=name) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def desargues_graph(create_using=None): + # This can also be defined w.r.t. LCF_graph + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, + 14, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 5, 19, 0, 2, 16, 1, 3, 11, 2, 4, 14, 3, 5, 9, 0, 4, 6, 5, 7, 15, 6, 8, + 18, 7, 9, 13, 4, 8, 10, 9, 11, 19, 2, 10, 12, 11, 13, 17, 8, 12, 14, 3, + 13, 15, 6, 14, 16, 1, 15, 17, 12, 16, 18, 7, 17, 19, 0, 10, 18, + ], + index_dtype, + ) + # fmt: on + if graph_class.is_multigraph(): + src_indices_extra = cp.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + index_dtype, + ) + dst_indices_extra = cp.array( + [5, 16, 11, 14, 9, 0, 15, 18, 13, 4, 19, 2, 17, 8, 3, 6, 1, 12, 7, 10], + index_dtype, + ) + src_indices = cp.hstack((src_indices, src_indices_extra)) + dst_indices = cp.hstack((dst_indices, dst_indices_extra)) + G = graph_class.from_coo(20, src_indices, dst_indices, name="Desargues Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def diamond_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_indices = cp.array([0, 0, 1, 1, 1, 2, 2, 2, 3, 3], index_dtype) + dst_indices = cp.array([1, 2, 0, 2, 3, 0, 1, 3, 1, 2], index_dtype) + G = graph_class.from_coo(4, src_indices, dst_indices, name="Diamond Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def dodecahedral_graph(create_using=None): + # This can also be defined w.r.t. LCF_graph + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, + 14, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 10, 19, 0, 2, 8, 1, 3, 6, 2, 4, 19, 3, 5, 17, 4, 6, 15, 2, 5, 7, 6, 8, + 14, 1, 7, 9, 8, 10, 13, 0, 9, 11, 10, 12, 18, 11, 13, 16, 9, 12, 14, 7, + 13, 15, 5, 14, 16, 12, 15, 17, 4, 16, 18, 11, 17, 19, 0, 3, 18, + ], + index_dtype, + ) + # fmt: on + if graph_class.is_multigraph(): + src_indices_extra = cp.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + index_dtype, + ) + dst_indices_extra = cp.array( + [10, 8, 6, 19, 17, 15, 2, 14, 1, 13, 0, 18, 16, 9, 7, 5, 12, 4, 11, 3], + index_dtype, + ) + src_indices = cp.hstack((src_indices, src_indices_extra)) + dst_indices = cp.hstack((dst_indices, dst_indices_extra)) + G = graph_class.from_coo(20, src_indices, dst_indices, name="Dodecahedral Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def frucht_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + src_indices = cp.array( + [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 10], + index_dtype, + ) + dst_indices = cp.array( + [1, 7, 2, 7, 3, 8, 4, 9, 5, 9, 6, 10, 0, 10, 11, 9, 11, 11], + index_dtype, + ) + else: + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, + 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 6, 7, 0, 2, 7, 1, 3, 8, 2, 4, 9, 3, 5, 9, 4, 6, 10, 0, 5, 10, 0, + 1, 11, 2, 9, 11, 3, 4, 8, 5, 6, 11, 7, 8, 10, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo(12, src_indices, dst_indices, name="Frucht Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def heawood_graph(create_using=None): + # This can also be defined w.r.t. LCF_graph + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 5, 13, 0, 2, 10, 1, 3, 7, 2, 4, 12, 3, 5, 9, 0, 4, 6, 5, 7, 11, 2, 6, + 8, 7, 9, 13, 4, 8, 10, 1, 9, 11, 6, 10, 12, 3, 11, 13, 0, 8, 12, + ], + index_dtype, + ) + # fmt: on + if graph_class.is_multigraph(): + src_indices_extra = cp.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + index_dtype, + ) + dst_indices_extra = cp.array( + [5, 10, 7, 12, 9, 0, 11, 2, 13, 4, 1, 6, 3, 8], + index_dtype, + ) + src_indices = cp.hstack((src_indices, src_indices_extra)) + dst_indices = cp.hstack((dst_indices, dst_indices_extra)) + G = graph_class.from_coo(14, src_indices, dst_indices, name="Heawood Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def house_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_indices = cp.array([0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4], index_dtype) + dst_indices = cp.array([1, 2, 0, 3, 0, 3, 4, 1, 2, 4, 2, 3], index_dtype) + G = graph_class.from_coo(5, src_indices, dst_indices, name="House Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def house_x_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_indices = cp.array( + [0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4], index_dtype + ) + dst_indices = cp.array( + [1, 2, 3, 0, 2, 3, 0, 1, 3, 4, 0, 1, 2, 4, 2, 3], index_dtype + ) + G = graph_class.from_coo( + 5, src_indices, dst_indices, name="House-with-X-inside Graph" + ) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def icosahedral_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, + 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, + 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 5, 7, 8, 11, 0, 2, 5, 6, 8, 1, 3, 6, 8, 9, 2, 4, 6, 9, 10, 3, 5, 6, + 10, 11, 0, 1, 4, 6, 11, 1, 2, 3, 4, 5, 0, 8, 9, 10, 11, 0, 1, 2, 7, 9, 2, + 3, 7, 8, 10, 3, 4, 7, 9, 11, 0, 4, 5, 7, 10, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo( + 12, src_indices, dst_indices, name="Platonic Icosahedral Graph" + ) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def krackhardt_kite_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, + 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 2, 3, 5, 0, 3, 4, 6, 0, 3, 5, 0, 1, 2, 4, 5, 6, 1, 3, 6, 0, 2, 3, 6, + 7, 1, 3, 4, 5, 7, 5, 6, 8, 7, 9, 8, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo( + 10, src_indices, dst_indices, name="Krackhardt Kite Social Network" + ) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def moebius_kantor_graph(create_using=None): + # This can also be defined w.r.t. LCF_graph + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, + 14, 14, 15, 15, 15, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 5, 15, 0, 2, 12, 1, 3, 7, 2, 4, 14, 3, 5, 9, 0, 4, 6, 5, 7, 11, 2, 6, + 8, 7, 9, 13, 4, 8, 10, 9, 11, 15, 6, 10, 12, 1, 11, 13, 8, 12, 14, 3, 13, + 15, 0, 10, 14, + ], + index_dtype, + ) + # fmt: on + if graph_class.is_multigraph(): + src_indices_extra = cp.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + index_dtype, + ) + dst_indices_extra = cp.array( + [5, 12, 7, 14, 9, 0, 11, 2, 13, 4, 15, 6, 1, 8, 3, 10], + index_dtype, + ) + src_indices = cp.hstack((src_indices, src_indices_extra)) + dst_indices = cp.hstack((dst_indices, dst_indices_extra)) + G = graph_class.from_coo(16, src_indices, dst_indices, name="Moebius-Kantor Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def octahedral_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + src_indices = cp.array( + [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5], + index_dtype, + ) + dst_indices = cp.array( + [1, 2, 3, 4, 0, 2, 3, 5, 0, 1, 4, 5, 0, 1, 4, 5, 0, 2, 3, 5, 1, 2, 3, 4], + index_dtype, + ) + G = graph_class.from_coo( + 6, src_indices, dst_indices, name="Platonic Octahedral Graph" + ) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def pappus_graph(): + # This can also be defined w.r.t. LCF_graph + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, + 14, 15, 15, 15, 16, 16, 16, 17, 17, 17, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 5, 17, 0, 2, 8, 1, 3, 13, 2, 4, 10, 3, 5, 15, 0, 4, 6, 5, 7, 11, 6, 8, + 14, 1, 7, 9, 8, 10, 16, 3, 9, 11, 6, 10, 12, 11, 13, 17, 2, 12, 14, 7, + 13, 15, 4, 14, 16, 9, 15, 17, 0, 12, 16, + ], + index_dtype, + ) + # fmt: on + return nxcg.Graph.from_coo(18, src_indices, dst_indices, name="Pappus Graph") + + +@networkx_algorithm +def petersen_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 4, 5, 0, 2, 6, 1, 3, 7, 2, 4, 8, 0, 3, 9, 0, 7, 8, 1, 8, 9, 2, 5, 9, + 3, 5, 6, 4, 6, 7, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo(10, src_indices, dst_indices, name="Petersen Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def sedgewick_maze_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + src_indices = cp.array([0, 0, 0, 1, 2, 3, 3, 4, 4, 4], index_dtype) + dst_indices = cp.array([2, 5, 7, 7, 6, 4, 5, 5, 6, 7], index_dtype) + else: + src_indices = cp.array( + [0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7], + index_dtype, + ) + dst_indices = cp.array( + [2, 5, 7, 7, 0, 6, 4, 5, 3, 5, 6, 7, 0, 3, 4, 2, 4, 0, 1, 4], + index_dtype, + ) + G = graph_class.from_coo(8, src_indices, dst_indices, name="Sedgewick Maze") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def tetrahedral_graph(create_using=None): + # This can also be defined w.r.t. complete_graph + graph_class, inplace = _create_using_class(create_using) + src_indices = cp.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3], index_dtype) + dst_indices = cp.array([1, 2, 3, 0, 2, 3, 0, 1, 3, 0, 1, 2], index_dtype) + name = ( + "Platonic Tetrahedral graph" + if _IS_NX32_OR_LESS + else "Platonic Tetrahedral Graph" + ) + G = graph_class.from_coo(4, src_indices, dst_indices, name=name) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def truncated_cube_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, + 14, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 20, 20, + 20, 21, 21, 21, 22, 22, 22, 23, 23, 23, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 2, 4, 0, 11, 14, 0, 3, 4, 2, 6, 8, 0, 2, 5, 4, 16, 18, 3, 7, 8, 6, 10, + 12, 3, 6, 9, 8, 17, 20, 7, 11, 12, 1, 10, 14, 7, 10, 13, 12, 21, 22, 1, + 11, 15, 14, 19, 23, 5, 17, 18, 9, 16, 20, 5, 16, 19, 15, 18, 23, 9, 17, + 21, 13, 20, 22, 13, 21, 23, 15, 19, 22, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo(24, src_indices, dst_indices, name="Truncated Cube Graph") + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def truncated_tetrahedron_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + src_indices = cp.array( + [0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 10], index_dtype + ) + dst_indices = cp.array( + [1, 2, 9, 2, 6, 3, 4, 11, 5, 11, 6, 7, 7, 8, 9, 10, 10, 11], index_dtype + ) + else: + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, + 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 2, 9, 0, 2, 6, 0, 1, 3, 2, 4, 11, 3, 5, 11, 4, 6, 7, 1, 5, 7, 5, + 6, 8, 7, 9, 10, 0, 8, 10, 8, 9, 11, 3, 4, 10, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo( + 12, src_indices, dst_indices, name="Truncated Tetrahedron Graph" + ) + if inplace: + return create_using._become(G) + return G + + +@networkx_algorithm +def tutte_graph(create_using=None): + graph_class, inplace = _create_using_class(create_using) + if graph_class.is_directed(): + raise nx.NetworkXError("Directed Graph not supported") + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, + 14, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 20, 20, + 20, 21, 21, 21, 22, 22, 22, 23, 23, 23, 24, 24, 24, 25, 25, 25, 26, 26, + 26, 27, 27, 27, 28, 28, 28, 29, 29, 29, 30, 30, 30, 31, 31, 31, 32, 32, + 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 36, 36, 37, 37, 37, 38, 38, + 38, 39, 39, 39, 40, 40, 40, 41, 41, 41, 42, 42, 42, 43, 43, 43, 44, 44, + 44, 45, 45, 45, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 2, 3, 0, 4, 26, 0, 10, 11, 0, 18, 19, 1, 5, 33, 4, 6, 29, 5, 7, 27, 6, + 8, 14, 7, 9, 38, 8, 10, 37, 2, 9, 39, 2, 12, 39, 11, 13, 35, 12, 14, 15, + 7, 13, 34, 13, 16, 22, 15, 17, 44, 16, 18, 43, 3, 17, 45, 3, 20, 45, 19, + 21, 41, 20, 22, 23, 15, 21, 40, 21, 24, 27, 23, 25, 32, 24, 26, 31, 1, + 25, 33, 6, 23, 28, 27, 29, 32, 5, 28, 30, 29, 31, 33, 25, 30, 32, 24, 28, + 31, 4, 26, 30, 14, 35, 38, 12, 34, 36, 35, 37, 39, 9, 36, 38, 8, 34, 37, + 10, 11, 36, 22, 41, 44, 20, 40, 42, 41, 43, 45, 17, 42, 44, 16, 40, 43, + 18, 19, 42, + ], + index_dtype, + ) + # fmt: on + G = graph_class.from_coo(46, src_indices, dst_indices, name="Tutte's Graph") + if inplace: + return create_using._become(G) + return G diff --git a/python/nx-cugraph/nx_cugraph/generators/social.py b/python/nx-cugraph/nx_cugraph/generators/social.py new file mode 100644 index 00000000000..3c936d07af3 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/generators/social.py @@ -0,0 +1,294 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import numpy as np + +import nx_cugraph as nxcg + +from ..utils import index_dtype, networkx_algorithm + +__all__ = [ + "davis_southern_women_graph", + "florentine_families_graph", + "karate_club_graph", + "les_miserables_graph", +] + + +@networkx_algorithm +def davis_southern_women_graph(): + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, + 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, + 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, + 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 15, 15, + 16, 16, 17, 17, 18, 18, 18, 19, 19, 19, 20, 20, 20, 20, 20, 20, 21, 21, + 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 27, 27, 27, 27, 27, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 30, 30, 30, + 31, 31, 31, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 18, 19, 20, 21, 22, 23, 25, 26, 18, 19, 20, 22, 23, 24, 25, 19, 20, 21, + 22, 23, 24, 25, 26, 18, 20, 21, 22, 23, 24, 25, 20, 21, 22, 24, 20, 22, + 23, 25, 22, 23, 24, 25, 23, 25, 26, 22, 24, 25, 26, 24, 25, 26, 29, 25, + 26, 27, 29, 25, 26, 27, 29, 30, 31, 24, 25, 26, 27, 29, 30, 31, 23, 24, + 26, 27, 28, 29, 30, 31, 24, 25, 27, 28, 29, 25, 26, 26, 28, 26, 28, 0, 1, + 3, 0, 1, 2, 0, 1, 2, 3, 4, 5, 0, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 8, 0, 1, + 2, 3, 5, 6, 7, 13, 1, 2, 3, 4, 6, 8, 9, 12, 13, 14, 0, 1, 2, 3, 5, 6, 7, + 8, 9, 10, 11, 12, 14, 15, 0, 2, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 10, + 11, 12, 13, 14, 13, 14, 16, 17, 9, 10, 11, 12, 13, 14, 11, 12, 13, 11, + 12, 13, + ], + index_dtype, + ) + bipartite = cp.array( + [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ], + np.int8, + ) + women = [ + "Evelyn Jefferson", "Laura Mandeville", "Theresa Anderson", "Brenda Rogers", + "Charlotte McDowd", "Frances Anderson", "Eleanor Nye", "Pearl Oglethorpe", + "Ruth DeSand", "Verne Sanderson", "Myra Liddel", "Katherina Rogers", + "Sylvia Avondale", "Nora Fayette", "Helen Lloyd", "Dorothy Murchison", + "Olivia Carleton", "Flora Price", + ] + events = [ + "E1", "E2", "E3", "E4", "E5", "E6", "E7", "E8", "E9", "E10", "E11", "E12", + "E13", "E14", + ] + # fmt: on + return nxcg.Graph.from_coo( + 32, + src_indices, + dst_indices, + node_values={"bipartite": bipartite}, + id_to_key=women + events, + top=women, + bottom=events, + ) + + +@networkx_algorithm +def florentine_families_graph(): + # fmt: off + src_indices = cp.array( + [ + 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 8, + 9, 10, 10, 10, 11, 11, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 8, 5, 6, 8, 4, 8, 6, 10, 13, 2, 10, 13, 1, 1, 3, 7, 14, 6, 0, 1, 2, 11, + 12, 14, 12, 3, 4, 13, 8, 13, 14, 8, 9, 3, 4, 10, 11, 6, 8, 11, + ], + index_dtype, + ) + nodes = [ + "Acciaiuoli", "Albizzi", "Barbadori", "Bischeri", "Castellani", "Ginori", + "Guadagni", "Lamberteschi", "Medici", "Pazzi", "Peruzzi", "Ridolfi", + "Salviati", "Strozzi", "Tornabuoni" + ] + # fmt: on + return nxcg.Graph.from_coo(15, src_indices, dst_indices, id_to_key=nodes) + + +@networkx_algorithm +def karate_club_graph(): + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, + 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 10, 10, 10, 11, 12, 12, 13, + 13, 13, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 19, 20, + 20, 21, 21, 22, 22, 23, 23, 23, 23, 23, 24, 24, 24, 25, 25, 25, 26, 26, + 27, 27, 27, 27, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 17, 19, 21, 31, 0, 2, 3, 7, 13, + 17, 19, 21, 30, 0, 1, 3, 7, 8, 9, 13, 27, 28, 32, 0, 1, 2, 7, 12, 13, 0, + 6, 10, 0, 6, 10, 16, 0, 4, 5, 16, 0, 1, 2, 3, 0, 2, 30, 32, 33, 2, 33, + 0, 4, 5, 0, 0, 3, 0, 1, 2, 3, 33, 32, 33, 32, 33, 5, 6, 0, 1, 32, 33, 0, + 1, 33, 32, 33, 0, 1, 32, 33, 25, 27, 29, 32, 33, 25, 27, 31, 23, 24, 31, + 29, 33, 2, 23, 24, 33, 2, 31, 33, 23, 26, 32, 33, 1, 8, 32, 33, 0, 24, + 25, 28, 32, 33, 2, 8, 14, 15, 18, 20, 22, 23, 29, 30, 31, 33, 8, 9, 13, + 14, 15, 18, 19, 20, 22, 23, 26, 27, 28, 29, 30, 31, 32, + ], + index_dtype, + ) + weights = cp.array( + [ + 4, 5, 3, 3, 3, 3, 2, 2, 2, 3, 1, 3, 2, 2, 2, 2, 4, 6, 3, 4, 5, 1, 2, 2, + 2, 5, 6, 3, 4, 5, 1, 3, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 5, 3, 3, + 3, 2, 5, 3, 2, 4, 4, 3, 2, 5, 3, 3, 4, 1, 2, 2, 3, 3, 3, 1, 3, 3, 5, 3, + 3, 3, 3, 2, 3, 4, 3, 3, 2, 1, 1, 2, 2, 2, 1, 3, 1, 2, 2, 2, 3, 5, 4, 3, + 5, 4, 2, 3, 2, 5, 2, 7, 4, 2, 2, 4, 3, 4, 2, 2, 2, 3, 4, 4, 2, 2, 3, 3, + 3, 2, 2, 7, 2, 4, 4, 2, 3, 3, 3, 1, 3, 2, 5, 4, 3, 4, 5, 4, 2, 3, 2, 4, + 2, 1, 1, 3, 4, 2, 4, 2, 2, 3, 4, 5, + ], + np.int8, + ) + # For now, cupy doesn't handle str dtypes and we primarily handle cupy arrays. + # We try to support numpy arrays for node values, so let's use numpy here. + clubs = np.array([ + "Mr. Hi", "Mr. Hi", "Mr. Hi", "Mr. Hi", "Mr. Hi", "Mr. Hi", "Mr. Hi", + "Mr. Hi", "Mr. Hi", "Officer", "Mr. Hi", "Mr. Hi", "Mr. Hi", "Mr. Hi", + "Officer", "Officer", "Mr. Hi", "Mr. Hi", "Officer", "Mr. Hi", "Officer", + "Mr. Hi", "Officer", "Officer", "Officer", "Officer", "Officer", "Officer", + "Officer", "Officer", "Officer", "Officer", "Officer", "Officer", + ]) + # fmt: on + return nxcg.Graph.from_coo( + 34, + src_indices, + dst_indices, + edge_values={"weight": weights}, + node_values={"club": clubs}, + name="Zachary's Karate Club", + ) + + +@networkx_algorithm +def les_miserables_graph(): + # fmt: off + src_indices = cp.array( + [ + 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 10, 10, + 10, 10, 10, 10, 11, 12, 12, 12, 12, 12, 12, 13, 13, 14, 14, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, + 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 23, 23, + 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, + 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28, 28, + 28, 28, 29, 29, 29, 29, 29, 29, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 38, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 42, 42, 42, 42, 42, + 42, 43, 44, 44, 44, 44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 46, + 46, 46, 46, 46, 46, 46, 47, 47, 48, 48, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 50, 50, 50, 51, 51, 51, 51, + 51, 51, 51, 52, 53, 53, 54, 55, 55, 55, 55, 55, 55, 55, 56, 56, 56, 57, + 57, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 60, 60, 61, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 64, + 65, 65, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 67, 68, 69, 69, 69, + 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, + 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, + 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 74, 74, 75, 75, 75, 76, 76, + 76, 76, 76, 76, 76, + ], + index_dtype, + ) + dst_indices = cp.array( + [ + 25, 58, 70, 9, 15, 25, 31, 37, 39, 58, 59, 70, 73, 6, 17, 21, 24, 30, 31, + 35, 40, 46, 49, 55, 67, 8, 10, 12, 16, 27, 39, 42, 73, 34, 49, 23, 26, + 27, 29, 44, 71, 76, 2, 17, 21, 24, 30, 31, 35, 40, 46, 49, 55, 67, 73, + 70, 3, 10, 12, 16, 42, 73, 1, 15, 25, 31, 37, 59, 70, 3, 8, 12, 16, 42, + 73, 62, 3, 8, 10, 16, 42, 73, 14, 31, 13, 31, 1, 9, 24, 25, 37, 39, 58, + 59, 70, 73, 3, 8, 10, 12, 42, 73, 2, 6, 21, 24, 30, 31, 35, 40, 46, 49, + 67, 34, 39, 45, 49, 51, 58, 70, 71, 72, 73, 75, 62, 62, 2, 6, 17, 24, 25, + 30, 31, 35, 40, 46, 49, 55, 67, 62, 5, 26, 27, 29, 44, 71, 76, 2, 6, 15, + 17, 21, 30, 31, 35, 39, 40, 46, 49, 55, 67, 73, 0, 1, 9, 15, 21, 37, 46, + 49, 58, 59, 70, 5, 23, 27, 29, 44, 71, 76, 3, 5, 23, 26, 29, 39, 44, 48, + 58, 65, 69, 70, 71, 73, 76, 36, 39, 60, 73, 5, 23, 26, 27, 44, 71, 76, 2, + 6, 17, 21, 24, 31, 35, 40, 46, 49, 67, 1, 2, 6, 9, 13, 14, 17, 21, 24, + 30, 35, 37, 39, 40, 46, 49, 53, 55, 59, 67, 70, 73, 62, 73, 4, 18, 45, + 47, 49, 51, 73, 2, 6, 17, 21, 24, 30, 31, 40, 55, 67, 28, 1, 9, 15, 25, + 31, 39, 58, 59, 70, 73, 73, 1, 3, 15, 18, 24, 27, 28, 31, 37, 58, 59, 69, + 70, 72, 73, 74, 75, 2, 6, 17, 21, 24, 30, 31, 35, 46, 49, 55, 67, 53, 3, + 8, 10, 12, 16, 73, 73, 5, 23, 26, 27, 29, 71, 76, 18, 34, 49, 51, 2, 6, + 17, 21, 24, 25, 30, 31, 40, 49, 61, 34, 58, 27, 73, 2, 4, 6, 17, 18, 21, + 24, 25, 30, 31, 34, 40, 45, 46, 51, 66, 70, 71, 73, 56, 62, 73, 18, 34, + 45, 49, 52, 57, 73, 51, 31, 41, 73, 2, 6, 21, 24, 31, 35, 40, 50, 62, 73, + 51, 66, 0, 1, 15, 18, 25, 27, 37, 39, 47, 70, 73, 1, 9, 15, 25, 31, 37, + 39, 70, 73, 28, 73, 46, 11, 19, 20, 22, 32, 50, 56, 63, 64, 73, 62, 62, + 27, 69, 49, 57, 70, 2, 6, 17, 21, 24, 30, 31, 35, 40, 73, 27, 39, 65, 73, + 0, 1, 7, 9, 15, 18, 25, 27, 31, 37, 39, 49, 58, 59, 66, 73, 5, 18, 23, + 26, 27, 29, 44, 49, 76, 18, 39, 73, 1, 3, 6, 8, 10, 12, 15, 16, 18, 24, + 27, 28, 31, 33, 34, 37, 38, 39, 42, 43, 48, 49, 50, 51, 54, 56, 58, 59, + 60, 62, 68, 69, 70, 72, 74, 75, 39, 73, 18, 39, 73, 5, 23, 26, 27, 29, + 44, 71, + ], + index_dtype, + ) + weights = cp.array( + [ + 2, 1, 2, 3, 4, 1, 1, 6, 2, 1, 2, 6, 1, 4, 5, 6, 4, 3, 5, 1, 5, 2, 1, 1, + 2, 1, 2, 1, 1, 1, 1, 2, 2, 1, 1, 3, 4, 3, 4, 4, 4, 3, 4, 9, 12, 10, 6, 5, + 3, 7, 1, 5, 1, 2, 1, 1, 1, 2, 2, 2, 2, 2, 3, 1, 1, 1, 3, 1, 3, 2, 2, 2, + 2, 3, 3, 1, 1, 2, 2, 2, 2, 2, 3, 2, 3, 2, 4, 1, 1, 1, 4, 1, 1, 2, 4, 1, + 1, 2, 2, 2, 2, 2, 5, 9, 13, 15, 5, 6, 1, 5, 2, 5, 2, 3, 1, 1, 21, 2, 4, + 1, 1, 2, 31, 1, 2, 1, 6, 12, 13, 17, 1, 6, 7, 2, 5, 2, 9, 1, 3, 1, 3, 3, + 4, 5, 3, 3, 4, 4, 10, 1, 15, 17, 6, 7, 3, 6, 5, 1, 7, 1, 4, 4, 2, 1, 1, + 1, 1, 1, 1, 5, 2, 1, 3, 4, 3, 3, 3, 4, 4, 3, 1, 3, 4, 3, 4, 5, 3, 2, 2, + 1, 2, 1, 3, 9, 4, 2, 1, 3, 8, 4, 5, 3, 4, 3, 3, 4, 3, 6, 5, 6, 6, 2, 1, + 5, 1, 1, 2, 1, 5, 5, 1, 2, 2, 6, 7, 7, 2, 1, 1, 1, 3, 1, 4, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 3, 1, 1, 12, 9, 2, 1, 3, 1, 2, 3, 1, 1, 2, 1, 1, 2, 6, 3, + 4, 1, 1, 1, 1, 2, 5, 1, 1, 2, 1, 1, 1, 6, 5, 1, 1, 1, 1, 1, 1, 5, 1, 17, + 1, 1, 5, 7, 5, 5, 5, 5, 3, 2, 1, 2, 1, 2, 1, 2, 2, 3, 2, 2, 3, 1, 4, 3, + 4, 3, 3, 4, 3, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 3, 1, 1, 2, 1, + 1, 1, 5, 5, 21, 9, 7, 5, 1, 4, 12, 2, 1, 1, 6, 1, 2, 1, 19, 6, 8, 3, 2, + 9, 2, 6, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 10, 3, 1, 1, 1, 1, + 1, 4, 2, 2, 1, 1, 1, 13, 7, 2, 1, 2, 1, 1, 2, 1, 1, 1, 3, 1, 3, 1, 2, 1, + 1, 1, 8, 10, 1, 1, 5, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 3, 4, 2, 1, 1, 2, 1, + 2, 1, 2, 3, 2, 6, 1, 3, 4, 1, 3, 1, 1, 5, 5, 2, 13, 1, 1, 12, 4, 1, 3, 4, + 3, 3, 4, 1, 3, 2, 1, 1, 1, 2, 1, 2, 3, 2, 1, 2, 31, 4, 9, 8, 1, 1, 2, 1, + 1, 17, 3, 1, 1, 19, 3, 2, 1, 3, 7, 1, 1, 5, 1, 3, 12, 1, 2, 3, 1, 2, 1, + 1, 3, 3, 4, 3, 4, 4, 3, 3, + ], + np.int8, + ) + nodes = [ + "Anzelma", "Babet", "Bahorel", "Bamatabois", "BaronessT", "Blacheville", + "Bossuet", "Boulatruelle", "Brevet", "Brujon", "Champmathieu", + "Champtercier", "Chenildieu", "Child1", "Child2", "Claquesous", + "Cochepaille", "Combeferre", "Cosette", "Count", "CountessDeLo", + "Courfeyrac", "Cravatte", "Dahlia", "Enjolras", "Eponine", "Fameuil", + "Fantine", "Fauchelevent", "Favourite", "Feuilly", "Gavroche", "Geborand", + "Gervais", "Gillenormand", "Grantaire", "Gribier", "Gueulemer", "Isabeau", + "Javert", "Joly", "Jondrette", "Judge", "Labarre", "Listolier", + "LtGillenormand", "Mabeuf", "Magnon", "Marguerite", "Marius", + "MlleBaptistine", "MlleGillenormand", "MlleVaubois", "MmeBurgon", "MmeDeR", + "MmeHucheloup", "MmeMagloire", "MmePontmercy", "MmeThenardier", + "Montparnasse", "MotherInnocent", "MotherPlutarch", "Myriel", "Napoleon", + "OldMan", "Perpetue", "Pontmercy", "Prouvaire", "Scaufflaire", "Simplice", + "Thenardier", "Tholomyes", "Toussaint", "Valjean", "Woman1", "Woman2", + "Zephine", + ] + # fmt: on + return nxcg.Graph.from_coo( + 77, src_indices, dst_indices, edge_values={"weight": weights}, id_to_key=nodes + ) diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 2ad23acd940..be6b3596030 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -12,6 +12,8 @@ # limitations under the License. from __future__ import annotations +import sys + import networkx as nx import nx_cugraph as nxcg @@ -63,18 +65,36 @@ def key(testpath): no_weights = "weighted implementation not currently supported" no_multigraph = "multigraphs not currently supported" louvain_different = "Louvain may be different due to RNG" + no_string_dtype = "string edge values not currently supported" xfail = {} from packaging.version import parse nxver = parse(nx.__version__) - if nxver.major == 3 and nxver.minor in {0, 1}: + + if nxver.major == 3 and nxver.minor <= 2: + # Networkx versions prior to 3.2.1 have tests written to expect + # sp.sparse.linalg.ArpackNoConvergence exceptions raised on no + # convergence in HITS. Newer versions since the merge of + # https://github.com/networkx/networkx/pull/7084 expect + # nx.PowerIterationFailedConvergence, which is what nx_cugraph.hits + # raises, so we mark them as xfail for previous versions of NX. + xfail.update( + { + key( + "test_hits.py:TestHITS.test_hits_not_convergent" + ): "nx_cugraph.hits raises updated exceptions not caught in " + "these tests", + } + ) + + if nxver.major == 3 and nxver.minor <= 1: # MAINT: networkx 3.0, 3.1 # NetworkX 3.2 added the ability to "fallback to nx" if backend algorithms # raise NotImplementedError or `can_run` returns False. The tests below # exercise behavior we have not implemented yet, so we mark them as xfail - # for previous versions of NetworkX. + # for previous versions of NX. xfail.update( { key( @@ -174,14 +194,68 @@ def key(testpath): ): louvain_different, key("test_louvain.py:test_none_weight_param"): louvain_different, key("test_louvain.py:test_multigraph"): louvain_different, + # See networkx#6630 + key( + "test_louvain.py:test_undirected_selfloops" + ): "self-loops not handled in Louvain", } ) + if sys.version_info[:2] == (3, 9): + # This test is sensitive to RNG, which depends on Python version + xfail[ + key("test_louvain.py:test_threshold") + ] = "Louvain does not support seed parameter" + if nxver.major == 3 and nxver.minor >= 2: + xfail.update( + { + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multi_attr_incl_target" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multidigraph_and_edge_attr" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_int_attr_name" + ): no_string_dtype, + } + ) + if nxver.minor == 2: + different_iteration_order = "Different graph data iteration order" + xfail.update( + { + key( + "test_cycles.py:TestMinimumCycleBasis." + "test_gh6787_and_edge_attribute_names" + ): different_iteration_order, + key( + "test_euler.py:TestEulerianCircuit." + "test_eulerian_circuit_cycle" + ): different_iteration_order, + key( + "test_gml.py:TestGraph.test_special_float_label" + ): different_iteration_order, + } + ) + + too_slow = "Too slow to run" + maybe_oom = "out of memory in CI" + skip = { + key("test_tree_isomorphism.py:test_positive"): too_slow, + key("test_tree_isomorphism.py:test_negative"): too_slow, + key("test_efficiency.py:TestEfficiency.test_using_ego_graph"): maybe_oom, + } for item in items: kset = set(item.keywords) for (test_name, keywords), reason in xfail.items(): if item.name == test_name and keywords.issubset(kset): item.add_marker(pytest.mark.xfail(reason=reason)) + for (test_name, keywords), reason in skip.items(): + if item.name == test_name and keywords.issubset(kset): + item.add_marker(pytest.mark.skip(reason=reason)) @classmethod def can_run(cls, name, args, kwargs): @@ -189,10 +263,4 @@ def can_run(cls, name, args, kwargs): This is a proposed API to add to networkx dispatching machinery and may change. """ - return ( - hasattr(cls, name) - and getattr(cls, name).can_run(*args, **kwargs) - # We don't support MultiGraphs yet - and not any(isinstance(x, nx.MultiGraph) for x in args) - and not any(isinstance(x, nx.MultiGraph) for x in kwargs.values()) - ) + return hasattr(cls, name) and getattr(cls, name).can_run(*args, **kwargs) diff --git a/python/nx-cugraph/nx_cugraph/tests/bench_convert.py b/python/nx-cugraph/nx_cugraph/tests/bench_convert.py index 7e6278661c2..2eb432230eb 100644 --- a/python/nx-cugraph/nx_cugraph/tests/bench_convert.py +++ b/python/nx-cugraph/nx_cugraph/tests/bench_convert.py @@ -39,6 +39,8 @@ gpubenchmark = pytest_benchmark.plugin.benchmark +CREATE_USING = [nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph] + def _bench_helper(gpubenchmark, N, attr_kind, create_using, method): G = method(N, create_using=create_using) @@ -103,7 +105,7 @@ def _bench_helper_scipy(gpubenchmark, N, attr_kind, create_using, method, fmt): None, ], ) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) def bench_cycle_graph(gpubenchmark, N, attr_kind, create_using): _bench_helper(gpubenchmark, N, attr_kind, create_using, nx.cycle_graph) @@ -111,7 +113,7 @@ def bench_cycle_graph(gpubenchmark, N, attr_kind, create_using): @pytest.mark.skipif("not cugraph") @pytest.mark.parametrize("N", [1, 10**6]) @pytest.mark.parametrize("attr_kind", ["full", None]) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) @pytest.mark.parametrize("do_renumber", [True, False]) def bench_cycle_graph_cugraph(gpubenchmark, N, attr_kind, create_using, do_renumber): if N == 1 and not do_renumber: @@ -124,7 +126,7 @@ def bench_cycle_graph_cugraph(gpubenchmark, N, attr_kind, create_using, do_renum @pytest.mark.skipif("not scipy") @pytest.mark.parametrize("N", [1, 10**6]) @pytest.mark.parametrize("attr_kind", ["full", None]) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) @pytest.mark.parametrize("fmt", ["coo", "csr"]) def bench_cycle_graph_scipy(gpubenchmark, N, attr_kind, create_using, fmt): _bench_helper_scipy(gpubenchmark, N, attr_kind, create_using, nx.cycle_graph, fmt) @@ -143,14 +145,14 @@ def bench_cycle_graph_scipy(gpubenchmark, N, attr_kind, create_using, fmt): None, ], ) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) def bench_complete_graph_edgedata(gpubenchmark, N, attr_kind, create_using): _bench_helper(gpubenchmark, N, attr_kind, create_using, nx.complete_graph) @pytest.mark.parametrize("N", [3000]) @pytest.mark.parametrize("attr_kind", [None]) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) def bench_complete_graph_noedgedata(gpubenchmark, N, attr_kind, create_using): _bench_helper(gpubenchmark, N, attr_kind, create_using, nx.complete_graph) @@ -158,7 +160,7 @@ def bench_complete_graph_noedgedata(gpubenchmark, N, attr_kind, create_using): @pytest.mark.skipif("not cugraph") @pytest.mark.parametrize("N", [1, 1500]) @pytest.mark.parametrize("attr_kind", ["full", None]) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) @pytest.mark.parametrize("do_renumber", [True, False]) def bench_complete_graph_cugraph(gpubenchmark, N, attr_kind, create_using, do_renumber): if N == 1 and not do_renumber: @@ -171,7 +173,7 @@ def bench_complete_graph_cugraph(gpubenchmark, N, attr_kind, create_using, do_re @pytest.mark.skipif("not scipy") @pytest.mark.parametrize("N", [1, 1500]) @pytest.mark.parametrize("attr_kind", ["full", None]) -@pytest.mark.parametrize("create_using", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize("create_using", CREATE_USING) @pytest.mark.parametrize("fmt", ["coo", "csr"]) def bench_complete_graph_scipy(gpubenchmark, N, attr_kind, create_using, fmt): _bench_helper_scipy( diff --git a/python/nx-cugraph/nx_cugraph/tests/test_convert.py b/python/nx-cugraph/nx_cugraph/tests/test_convert.py index ba3cd7aaee1..1a71b796861 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_convert.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_convert.py @@ -18,7 +18,9 @@ from nx_cugraph import interface -@pytest.mark.parametrize("graph_class", [nx.Graph, nx.DiGraph]) +@pytest.mark.parametrize( + "graph_class", [nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph] +) @pytest.mark.parametrize( "kwargs", [ @@ -48,9 +50,10 @@ def test_convert_empty(graph_class, kwargs): assert G.graph == Gcg.graph == H.graph == {} -def test_convert(): +@pytest.mark.parametrize("graph_class", [nx.Graph, nx.MultiGraph]) +def test_convert(graph_class): # FIXME: can we break this into smaller tests? - G = nx.Graph() + G = graph_class() G.add_edge(0, 1, x=2) G.add_node(0, foo=10) G.add_node(1, foo=20, bar=100) @@ -68,8 +71,8 @@ def test_convert(): ]: # All edges have "x" attribute, so all kwargs are equivalent Gcg = nxcg.from_networkx(G, **kwargs) - cp.testing.assert_array_equal(Gcg.row_indices, [0, 1]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 0]) + cp.testing.assert_array_equal(Gcg.src_indices, [0, 1]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 0]) cp.testing.assert_array_equal(Gcg.edge_values["x"], [2, 2]) assert len(Gcg.edge_values) == 1 assert Gcg.edge_masks == {} @@ -83,43 +86,52 @@ def test_convert(): # Structure-only graph (no edge attributes) Gcg = nxcg.from_networkx(G, preserve_node_attrs=True) - cp.testing.assert_array_equal(Gcg.row_indices, [0, 1]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 0]) + cp.testing.assert_array_equal(Gcg.src_indices, [0, 1]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 0]) cp.testing.assert_array_equal(Gcg.node_values["foo"], [10, 20]) assert Gcg.edge_values == Gcg.edge_masks == {} H = nxcg.to_networkx(Gcg) - assert set(G.edges) == set(H.edges) == {(0, 1)} + if G.is_multigraph(): + assert set(G.edges) == set(H.edges) == {(0, 1, 0)} + else: + assert set(G.edges) == set(H.edges) == {(0, 1)} assert G.nodes == H.nodes # Fill completely missing attribute with default value Gcg = nxcg.from_networkx(G, edge_attrs={"y": 0}) - cp.testing.assert_array_equal(Gcg.row_indices, [0, 1]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 0]) + cp.testing.assert_array_equal(Gcg.src_indices, [0, 1]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 0]) cp.testing.assert_array_equal(Gcg.edge_values["y"], [0, 0]) assert len(Gcg.edge_values) == 1 assert Gcg.edge_masks == Gcg.node_values == Gcg.node_masks == {} H = nxcg.to_networkx(Gcg) assert list(H.edges(data=True)) == [(0, 1, {"y": 0})] + if Gcg.is_multigraph(): + assert set(H.edges) == {(0, 1, 0)} # If attribute is completely missing (and no default), then just ignore it Gcg = nxcg.from_networkx(G, edge_attrs={"y": None}) - cp.testing.assert_array_equal(Gcg.row_indices, [0, 1]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 0]) + cp.testing.assert_array_equal(Gcg.src_indices, [0, 1]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 0]) assert sorted(Gcg.edge_values) == sorted(Gcg.edge_masks) == [] H = nxcg.to_networkx(Gcg) assert list(H.edges(data=True)) == [(0, 1, {})] + if Gcg.is_multigraph(): + assert set(H.edges) == {(0, 1, 0)} G.add_edge(0, 2) # Some edges are missing 'x' attribute; need to use a mask for kwargs in [{"preserve_edge_attrs": True}, {"edge_attrs": {"x": None}}]: Gcg = nxcg.from_networkx(G, **kwargs) - cp.testing.assert_array_equal(Gcg.row_indices, [0, 0, 1, 2]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 2, 0, 0]) + cp.testing.assert_array_equal(Gcg.src_indices, [0, 0, 1, 2]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 2, 0, 0]) assert sorted(Gcg.edge_values) == sorted(Gcg.edge_masks) == ["x"] cp.testing.assert_array_equal(Gcg.edge_masks["x"], [True, False, True, False]) cp.testing.assert_array_equal(Gcg.edge_values["x"][Gcg.edge_masks["x"]], [2, 2]) H = nxcg.to_networkx(Gcg) assert list(H.edges(data=True)) == [(0, 1, {"x": 2}), (0, 2, {})] + if Gcg.is_multigraph(): + assert set(H.edges) == {(0, 1, 0), (0, 2, 0)} with pytest.raises(KeyError, match="x"): nxcg.from_networkx(G, edge_attrs={"x": nxcg.convert.REQUIRED}) @@ -131,7 +143,7 @@ def test_convert(): nxcg.from_networkx(G, node_attrs={"bar": ...}) # Now for something more complicated... - G = nx.Graph() + G = graph_class() G.add_edge(10, 20, x=1) G.add_edge(10, 30, x=2, y=1.5) G.add_node(10, foo=100) @@ -147,9 +159,9 @@ def test_convert(): {"edge_attrs": {"x": 0, "y": None}, "edge_dtypes": {"x": int, "y": float}}, ]: Gcg = nxcg.from_networkx(G, **kwargs) - assert Gcg.id_to_key == {0: 10, 1: 20, 2: 30} # Remap node IDs to 0, 1, ... - cp.testing.assert_array_equal(Gcg.row_indices, [0, 0, 1, 2]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 2, 0, 0]) + assert Gcg.id_to_key == [10, 20, 30] # Remap node IDs to 0, 1, ... + cp.testing.assert_array_equal(Gcg.src_indices, [0, 0, 1, 2]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 2, 0, 0]) cp.testing.assert_array_equal(Gcg.edge_values["x"], [1, 2, 1, 2]) assert sorted(Gcg.edge_masks) == ["y"] cp.testing.assert_array_equal(Gcg.edge_masks["y"], [False, True, False, True]) @@ -168,9 +180,9 @@ def test_convert(): {"node_attrs": {"foo": 0, "bar": None, "missing": None}}, ]: Gcg = nxcg.from_networkx(G, **kwargs) - assert Gcg.id_to_key == {0: 10, 1: 20, 2: 30} # Remap node IDs to 0, 1, ... - cp.testing.assert_array_equal(Gcg.row_indices, [0, 0, 1, 2]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 2, 0, 0]) + assert Gcg.id_to_key == [10, 20, 30] # Remap node IDs to 0, 1, ... + cp.testing.assert_array_equal(Gcg.src_indices, [0, 0, 1, 2]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 2, 0, 0]) cp.testing.assert_array_equal(Gcg.node_values["foo"], [100, 200, 300]) assert sorted(Gcg.node_masks) == ["bar"] cp.testing.assert_array_equal(Gcg.node_masks["bar"], [False, True, False]) @@ -189,9 +201,9 @@ def test_convert(): {"node_attrs": {"bar": 0, "foo": None}, "node_dtypes": int}, ]: Gcg = nxcg.from_networkx(G, **kwargs) - assert Gcg.id_to_key == {0: 10, 1: 20, 2: 30} # Remap node IDs to 0, 1, ... - cp.testing.assert_array_equal(Gcg.row_indices, [0, 0, 1, 2]) - cp.testing.assert_array_equal(Gcg.col_indices, [1, 2, 0, 0]) + assert Gcg.id_to_key == [10, 20, 30] # Remap node IDs to 0, 1, ... + cp.testing.assert_array_equal(Gcg.src_indices, [0, 0, 1, 2]) + cp.testing.assert_array_equal(Gcg.dst_indices, [1, 2, 0, 0]) cp.testing.assert_array_equal(Gcg.node_values["bar"], [0, 1000, 0]) assert Gcg.node_masks == {} @@ -201,3 +213,14 @@ def test_convert(): interface.BackendInterface.convert_from_nx(G, edge_attrs={"x": 1}, weight="x") with pytest.raises(TypeError, match="Expected networkx.Graph"): nxcg.from_networkx({}) + + +@pytest.mark.parametrize("graph_class", [nx.MultiGraph, nx.MultiDiGraph]) +def test_multigraph(graph_class): + G = graph_class() + G.add_edge(0, 1, "key1", x=10) + G.add_edge(0, 1, "key2", y=20) + Gcg = nxcg.from_networkx(G, preserve_edge_attrs=True) + H = nxcg.to_networkx(Gcg) + assert type(G) is type(H) + assert nx.utils.graphs_equal(G, H) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_generators.py b/python/nx-cugraph/nx_cugraph/tests/test_generators.py new file mode 100644 index 00000000000..511f8dcd8e2 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_generators.py @@ -0,0 +1,277 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import numpy as np +import pytest +from packaging.version import parse + +import nx_cugraph as nxcg + +nxver = parse(nx.__version__) + + +def assert_graphs_equal(Gnx, Gcg): + assert isinstance(Gnx, nx.Graph) + assert isinstance(Gcg, nxcg.Graph) + assert Gnx.number_of_nodes() == Gcg.number_of_nodes() + assert Gnx.number_of_edges() == Gcg.number_of_edges() + assert Gnx.is_directed() == Gcg.is_directed() + assert Gnx.is_multigraph() == Gcg.is_multigraph() + G = nxcg.to_networkx(Gcg) + rv = nx.utils.graphs_equal(G, Gnx) + if not rv: + print("GRAPHS ARE NOT EQUAL!") + assert sorted(G) == sorted(Gnx) + assert sorted(G._adj) == sorted(Gnx._adj) + assert sorted(G._node) == sorted(Gnx._node) + for k in sorted(G._adj): + print(k, sorted(G._adj[k]), sorted(Gnx._adj[k])) + print(nx.to_scipy_sparse_array(G).todense()) + print(nx.to_scipy_sparse_array(Gnx).todense()) + print(G.graph) + print(Gnx.graph) + assert rv + + +if nxver.major == 3 and nxver.minor < 2: + pytest.skip("Need NetworkX >=3.2 to test generators", allow_module_level=True) + + +def compare(name, create_using, *args, is_vanilla=False): + exc1 = exc2 = None + func = getattr(nx, name) + if isinstance(create_using, nxcg.Graph): + nx_create_using = nxcg.to_networkx(create_using) + elif isinstance(create_using, type) and issubclass(create_using, nxcg.Graph): + nx_create_using = create_using.to_networkx_class() + elif isinstance(create_using, nx.Graph): + nx_create_using = create_using.copy() + else: + nx_create_using = create_using + try: + if is_vanilla: + G = func(*args) + else: + G = func(*args, create_using=nx_create_using) + except Exception as exc: + exc1 = exc + try: + if is_vanilla: + Gcg = func(*args, backend="cugraph") + else: + Gcg = func(*args, create_using=create_using, backend="cugraph") + except ZeroDivisionError: + raise + except NotImplementedError as exc: + if name in {"complete_multipartite_graph"}: # nx.__version__[:3] <= "3.2" + return + exc2 = exc + except Exception as exc: + if exc1 is None: # pragma: no cover (debug) + raise + exc2 = exc + if exc1 is not None or exc2 is not None: + assert type(exc1) is type(exc2) + else: + assert_graphs_equal(G, Gcg) + + +N = list(range(-1, 5)) +CREATE_USING = [nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph] +COMPLETE_CREATE_USING = [ + nx.Graph, + nx.DiGraph, + nx.MultiGraph, + nx.MultiDiGraph, + nxcg.Graph, + nxcg.DiGraph, + nxcg.MultiGraph, + nxcg.MultiDiGraph, + # These raise NotImplementedError + # nx.Graph(), + # nx.DiGraph(), + # nx.MultiGraph(), + # nx.MultiDiGraph(), + nxcg.Graph(), + nxcg.DiGraph(), + nxcg.MultiGraph(), + nxcg.MultiDiGraph(), + None, + object, # Bad input + 7, # Bad input +] +GENERATORS_NOARG = [ + # classic + "null_graph", + "trivial_graph", + # small + "bull_graph", + "chvatal_graph", + "cubical_graph", + "desargues_graph", + "diamond_graph", + "dodecahedral_graph", + "frucht_graph", + "heawood_graph", + "house_graph", + "house_x_graph", + "icosahedral_graph", + "krackhardt_kite_graph", + "moebius_kantor_graph", + "octahedral_graph", + "petersen_graph", + "sedgewick_maze_graph", + "tetrahedral_graph", + "truncated_cube_graph", + "truncated_tetrahedron_graph", + "tutte_graph", +] +GENERATORS_NOARG_VANILLA = [ + # classic + "complete_multipartite_graph", + # small + "pappus_graph", + # social + "davis_southern_women_graph", + "florentine_families_graph", + "karate_club_graph", + "les_miserables_graph", +] +GENERATORS_N = [ + # classic + "circular_ladder_graph", + "complete_graph", + "cycle_graph", + "empty_graph", + "ladder_graph", + "path_graph", + "star_graph", + "wheel_graph", +] +GENERATORS_M_N = [ + # classic + "barbell_graph", + "lollipop_graph", + "tadpole_graph", + # bipartite + "complete_bipartite_graph", +] +GENERATORS_M_N_VANILLA = [ + # classic + "complete_multipartite_graph", + "turan_graph", + # community + "caveman_graph", +] + + +@pytest.mark.parametrize("name", GENERATORS_NOARG) +@pytest.mark.parametrize("create_using", COMPLETE_CREATE_USING) +def test_generator_noarg(name, create_using): + print(name, create_using, type(create_using)) + if isinstance(create_using, nxcg.Graph) and name in { + # fmt: off + "bull_graph", "chvatal_graph", "cubical_graph", "diamond_graph", + "house_graph", "house_x_graph", "icosahedral_graph", "krackhardt_kite_graph", + "octahedral_graph", "petersen_graph", "truncated_cube_graph", "tutte_graph", + # fmt: on + }: + # The _raise_on_directed decorator used in networkx doesn't like our graphs. + if create_using.is_directed(): + with pytest.raises(AssertionError): + compare(name, create_using) + else: + with pytest.raises(TypeError): + compare(name, create_using) + else: + compare(name, create_using) + + +@pytest.mark.parametrize("name", GENERATORS_NOARG_VANILLA) +def test_generator_noarg_vanilla(name): + print(name) + compare(name, None, is_vanilla=True) + + +@pytest.mark.parametrize("name", GENERATORS_N) +@pytest.mark.parametrize("n", N) +@pytest.mark.parametrize("create_using", CREATE_USING) +def test_generator_n(name, n, create_using): + print(name, n, create_using) + compare(name, create_using, n) + + +@pytest.mark.parametrize("name", GENERATORS_N) +@pytest.mark.parametrize("n", [1, 4]) +@pytest.mark.parametrize("create_using", COMPLETE_CREATE_USING) +def test_generator_n_complete(name, n, create_using): + print(name, n, create_using) + compare(name, create_using, n) + + +@pytest.mark.parametrize("name", GENERATORS_M_N) +@pytest.mark.parametrize("create_using", CREATE_USING) +@pytest.mark.parametrize("m", N) +@pytest.mark.parametrize("n", N) +def test_generator_m_n(name, create_using, m, n): + print(name, m, n, create_using) + compare(name, create_using, m, n) + + +@pytest.mark.parametrize("name", GENERATORS_M_N_VANILLA) +@pytest.mark.parametrize("m", N) +@pytest.mark.parametrize("n", N) +def test_generator_m_n_vanilla(name, m, n): + print(name, m, n) + compare(name, None, m, n, is_vanilla=True) + + +@pytest.mark.parametrize("name", GENERATORS_M_N) +@pytest.mark.parametrize("create_using", COMPLETE_CREATE_USING) +@pytest.mark.parametrize("m", [4]) +@pytest.mark.parametrize("n", [4]) +def test_generator_m_n_complete(name, create_using, m, n): + print(name, m, n, create_using) + compare(name, create_using, m, n) + + +@pytest.mark.parametrize("name", GENERATORS_M_N_VANILLA) +@pytest.mark.parametrize("m", [4]) +@pytest.mark.parametrize("n", [4]) +def test_generator_m_n_complete_vanilla(name, m, n): + print(name, m, n) + compare(name, None, m, n, is_vanilla=True) + + +def test_bad_lollipop_graph(): + compare("lollipop_graph", None, [0, 1], [1, 2]) + + +def test_can_convert_karate_club(): + # Karate club graph has string node values. + # This really tests conversions, but it's here so we can use `assert_graphs_equal`. + G = nx.karate_club_graph() + G.add_node(0, foo="bar") # string dtype with a mask + G.add_node(1, object=object()) # haha + Gcg = nxcg.from_networkx(G, preserve_all_attrs=True) + assert_graphs_equal(G, Gcg) + Gnx = nxcg.to_networkx(Gcg) + assert nx.utils.graphs_equal(G, Gnx) + assert isinstance(Gcg.node_values["club"], np.ndarray) + assert Gcg.node_values["club"].dtype.kind == "U" + assert isinstance(Gcg.node_values["foo"], np.ndarray) + assert isinstance(Gcg.node_masks["foo"], np.ndarray) + assert Gcg.node_values["foo"].dtype.kind == "U" + assert isinstance(Gcg.node_values["object"], np.ndarray) + assert Gcg.node_values["object"].dtype.kind == "O" + assert isinstance(Gcg.node_masks["object"], np.ndarray) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py b/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py new file mode 100644 index 00000000000..92fe2360688 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import pytest + +import nx_cugraph as nxcg + + +@pytest.mark.parametrize( + "get_graph", [nx.florentine_families_graph, nx.les_miserables_graph] +) +def test_k_truss(get_graph): + Gnx = get_graph() + Gcg = nxcg.from_networkx(Gnx, preserve_all_attrs=True) + for k in range(6): + Hnx = nx.k_truss(Gnx, k) + Hcg = nxcg.k_truss(Gcg, k) + assert nx.utils.graphs_equal(Hnx, nxcg.to_networkx(Hcg)) + if Hnx.number_of_edges() == 0: + break diff --git a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py index ecfda1397db..a654ff343ed 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py @@ -31,6 +31,9 @@ def test_match_signature_and_names(): if is_nx_30_or_31 and name in {"louvain_communities"}: continue + if name not in nx_backends._registered_algorithms: + print(f"{name} not dispatched from networkx") + continue dispatchable_func = nx_backends._registered_algorithms[name] # nx version >=3.2 uses orig_func, version >=3.0,<3.2 uses _orig_func if is_nx_30_or_31: diff --git a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py new file mode 100644 index 00000000000..a8f189a4745 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import pytest + +import nx_cugraph as nxcg + + +@pytest.mark.parametrize("test_nxcugraph", [False, True]) +def test_get_edge_data(test_nxcugraph): + G = nx.MultiGraph() + G.add_edge(0, 1, 0, x=10) + G.add_edge(0, 1, 1, y=20) + G.add_edge(0, 2, "a", x=100) + G.add_edge(0, 2, "b", y=200) + G.add_edge(0, 3) + G.add_edge(0, 3) + if test_nxcugraph: + G = nxcg.MultiGraph(G) + default = object() + assert G.get_edge_data(0, 0, default=default) is default + assert G.get_edge_data("a", "b", default=default) is default + assert G.get_edge_data(0, 1, 2, default=default) is default + assert G.get_edge_data(-1, 1, default=default) is default + assert G.get_edge_data(0, 1, 0, default=default) == {"x": 10} + assert G.get_edge_data(0, 1, 1, default=default) == {"y": 20} + assert G.get_edge_data(0, 1, default=default) == {0: {"x": 10}, 1: {"y": 20}} + assert G.get_edge_data(0, 2, "a", default=default) == {"x": 100} + assert G.get_edge_data(0, 2, "b", default=default) == {"y": 200} + assert G.get_edge_data(0, 2, default=default) == {"a": {"x": 100}, "b": {"y": 200}} + assert G.get_edge_data(0, 3, 0, default=default) == {} + assert G.get_edge_data(0, 3, 1, default=default) == {} + assert G.get_edge_data(0, 3, 2, default=default) is default + assert G.get_edge_data(0, 3, default=default) == {0: {}, 1: {}} + assert G.has_edge(0, 1) + assert G.has_edge(0, 1, 0) + assert G.has_edge(0, 1, 1) + assert not G.has_edge(0, 1, 2) + assert not G.has_edge(0, 1, "a") + assert not G.has_edge(0, -1) + assert G.has_edge(0, 2) + assert G.has_edge(0, 2, "a") + assert G.has_edge(0, 2, "b") + assert not G.has_edge(0, 2, "c") + assert not G.has_edge(0, 2, 0) + assert G.has_edge(0, 3) + assert not G.has_edge(0, 0) + assert not G.has_edge(0, 0, 0) + + G = nx.MultiGraph() + G.add_edge(0, 1) + if test_nxcugraph: + G = nxcg.MultiGraph(G) + assert G.get_edge_data(0, 1, default=default) == {0: {}} + assert G.get_edge_data(0, 1, 0, default=default) == {} + assert G.get_edge_data(0, 1, 1, default=default) is default + assert G.get_edge_data(0, 1, "b", default=default) is default + assert G.get_edge_data(-1, 2, default=default) is default + assert G.has_edge(0, 1) + assert G.has_edge(0, 1, 0) + assert not G.has_edge(0, 1, 1) + assert not G.has_edge(0, 1, "a") diff --git a/python/nx-cugraph/nx_cugraph/tests/test_utils.py b/python/nx-cugraph/nx_cugraph/tests/test_utils.py new file mode 100644 index 00000000000..fdd0c91995c --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_utils.py @@ -0,0 +1,87 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import pytest + +from nx_cugraph.utils import _get_int_dtype + + +def test_get_int_dtype(): + uint8 = np.dtype(np.uint8) + uint16 = np.dtype(np.uint16) + uint32 = np.dtype(np.uint32) + uint64 = np.dtype(np.uint64) + # signed + cur = np.iinfo(np.int8) + for val in [cur.min, cur.min + 1, -1, 0, 1, cur.max - 1, cur.max]: + assert _get_int_dtype(val) == np.int8 + assert _get_int_dtype(val, signed=True) == np.int8 + if val >= 0: + assert _get_int_dtype(val, unsigned=True) == np.uint8 + assert _get_int_dtype(val + 1, unsigned=True) == np.uint8 + prev = cur + cur = np.iinfo(np.int16) + for val in [cur.min, cur.min + 1, prev.min - 1, prev.max + 1, cur.max - 1, cur.max]: + assert _get_int_dtype(val) != prev.dtype + assert _get_int_dtype(val, signed=True) == np.int16 + if val >= 0: + assert _get_int_dtype(val, unsigned=True) in {uint8, uint16} + assert _get_int_dtype(val + 1, unsigned=True) in {uint8, uint16} + prev = cur + cur = np.iinfo(np.int32) + for val in [cur.min, cur.min + 1, prev.min - 1, prev.max + 1, cur.max - 1, cur.max]: + assert _get_int_dtype(val) != prev.dtype + assert _get_int_dtype(val, signed=True) == np.int32 + if val >= 0: + assert _get_int_dtype(val, unsigned=True) in {uint16, uint32} + assert _get_int_dtype(val + 1, unsigned=True) in {uint16, uint32} + prev = cur + cur = np.iinfo(np.int64) + for val in [cur.min, cur.min + 1, prev.min - 1, prev.max + 1, cur.max - 1, cur.max]: + assert _get_int_dtype(val) != prev.dtype + assert _get_int_dtype(val, signed=True) == np.int64 + if val >= 0: + assert _get_int_dtype(val, unsigned=True) in {uint32, uint64} + assert _get_int_dtype(val + 1, unsigned=True) in {uint32, uint64} + with pytest.raises(ValueError, match="Value is too"): + _get_int_dtype(cur.min - 1, signed=True) + with pytest.raises(ValueError, match="Value is too"): + _get_int_dtype(cur.max + 1, signed=True) + + # unsigned + cur = np.iinfo(np.uint8) + for val in [0, 1, cur.max - 1, cur.max]: + assert _get_int_dtype(val) == (np.uint8 if val > 1 else np.int8) + assert _get_int_dtype(val, unsigned=True) == np.uint8 + assert _get_int_dtype(cur.max + 1) == np.int16 + cur = np.iinfo(np.uint16) + for val in [cur.max - 1, cur.max]: + assert _get_int_dtype(val, unsigned=True) == np.uint16 + assert _get_int_dtype(cur.max + 1) == np.int32 + cur = np.iinfo(np.uint32) + for val in [cur.max - 1, cur.max]: + assert _get_int_dtype(val, unsigned=True) == np.uint32 + assert _get_int_dtype(cur.max + 1) == np.int64 + cur = np.iinfo(np.uint64) + for val in [cur.max - 1, cur.max]: + assert _get_int_dtype(val, unsigned=True) == np.uint64 + with pytest.raises(ValueError, match="Value is incompatible"): + _get_int_dtype(cur.min - 1, unsigned=True) + with pytest.raises(ValueError, match="Value is too"): + _get_int_dtype(cur.max + 1, unsigned=True) + + # API + with pytest.raises(TypeError, match="incompatible"): + _get_int_dtype(7, signed=True, unsigned=True) + assert _get_int_dtype(7, signed=True, unsigned=False) == np.int8 + assert _get_int_dtype(7, signed=False, unsigned=True) == np.uint8 diff --git a/python/nx-cugraph/nx_cugraph/typing.py b/python/nx-cugraph/nx_cugraph/typing.py index d3045ab4656..b419a9085e0 100644 --- a/python/nx-cugraph/nx_cugraph/typing.py +++ b/python/nx-cugraph/nx_cugraph/typing.py @@ -15,6 +15,9 @@ from collections.abc import Hashable from typing import TypeVar +import cupy as cp +import numpy as np + AttrKey = TypeVar("AttrKey", bound=Hashable) EdgeKey = TypeVar("EdgeKey", bound=Hashable) NodeKey = TypeVar("NodeKey", bound=Hashable) @@ -23,3 +26,8 @@ NodeValue = TypeVar("NodeValue") IndexValue = TypeVar("IndexValue") Dtype = TypeVar("Dtype") + + +class any_ndarray: + def __class_getitem__(cls, item): + return cp.ndarray[item] | np.ndarray[item] diff --git a/python/nx-cugraph/nx_cugraph/utils/decorators.py b/python/nx-cugraph/nx_cugraph/utils/decorators.py index 0f15d236ecd..0048aee51bb 100644 --- a/python/nx-cugraph/nx_cugraph/utils/decorators.py +++ b/python/nx-cugraph/nx_cugraph/utils/decorators.py @@ -13,6 +13,7 @@ from __future__ import annotations from functools import partial, update_wrapper +from textwrap import dedent from networkx.utils.decorators import nodes_or_number, not_implemented_for @@ -65,7 +66,9 @@ def __new__( ) instance.extra_params = extra_params # The docstring on our function is added to the NetworkX docstring. - instance.extra_doc = func.__doc__ + instance.extra_doc = ( + dedent(func.__doc__.lstrip("\n").rstrip()) if func.__doc__ else None + ) # Copy __doc__ from NetworkX if instance.name in _registered_algorithms: instance.__doc__ = _registered_algorithms[instance.name].__doc__ diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 72e4094b8b7..e303375918d 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -12,43 +12,83 @@ # limitations under the License. from __future__ import annotations +import itertools import operator as op import sys from random import Random +from typing import TYPE_CHECKING, SupportsIndex import cupy as cp +import numpy as np -__all__ = ["_groupby", "_seed_to_int"] +if TYPE_CHECKING: + from ..typing import Dtype +try: + from itertools import pairwise # Python >=3.10 +except ImportError: -def _groupby(groups: cp.ndarray, values: cp.ndarray) -> dict[int, cp.ndarray]: + def pairwise(it): + it = iter(it) + for prev in it: + for cur in it: + yield (prev, cur) + prev = cur + + +__all__ = [ + "index_dtype", + "_groupby", + "_seed_to_int", + "_get_int_dtype", + "_get_float_dtype", + "_dtype_param", +] + +# This may switch to np.uint32 at some point +index_dtype = np.int32 + +# To add to `extra_params=` of `networkx_algorithm` +_dtype_param = { + "dtype : dtype or None, optional": ( + "The data type (np.float32, np.float64, or None) to use for the edge weights " + "in the algorithm. If None, then dtype is determined by the edge values." + ), +} + + +def _groupby( + groups: cp.ndarray, values: cp.ndarray, groups_are_canonical: bool = False +) -> dict[int, cp.ndarray]: """Perform a groupby operation given an array of group IDs and array of values. Parameters ---------- groups : cp.ndarray Array that holds the group IDs. - Group IDs are assumed to be consecutive integers from 0. values : cp.ndarray Array of values to be grouped according to groups. Must be the same size as groups array. + groups_are_canonical : bool, default False + Whether the group IDs are consecutive integers beginning with 0. Returns ------- dict with group IDs as keys and cp.ndarray as values. """ - # It would actually be easy to support groups that aren't consecutive integers, - # but let's wait until we need it to implement it. - sorted_groups = cp.argsort(groups) - sorted_values = values[sorted_groups] - rv = {} - start = 0 - for i, end in enumerate( - [*(cp.nonzero(cp.diff(groups[sorted_groups]))[0] + 1).tolist(), groups.size] - ): - rv[i] = sorted_values[start:end] - start = end - return rv + if groups.size == 0: + return {} + sort_indices = cp.argsort(groups) + sorted_groups = groups[sort_indices] + sorted_values = values[sort_indices] + prepend = 1 if groups_are_canonical else sorted_groups[0] + 1 + left_bounds = cp.nonzero(cp.diff(sorted_groups, prepend=prepend))[0] + boundaries = pairwise(itertools.chain(left_bounds.tolist(), [groups.size])) + if groups_are_canonical: + it = enumerate(boundaries) + else: + it = zip(sorted_groups[left_bounds].tolist(), boundaries) + return {group: sorted_values[start:end] for group, (start, end) in it} def _seed_to_int(seed: int | Random | None) -> int: @@ -58,3 +98,79 @@ def _seed_to_int(seed: int | Random | None) -> int: if isinstance(seed, Random): return seed.randint(0, sys.maxsize) return op.index(seed) # Ensure seed is integral + + +def _get_int_dtype( + val: SupportsIndex, *, signed: bool | None = None, unsigned: bool | None = None +): + """Determine the smallest integer dtype that can store the integer ``val``. + + If signed or unsigned are unspecified, then signed integers are preferred + unless the value can be represented by a smaller unsigned integer. + + Raises + ------ + ValueError : If the value cannot be represented with an int dtype. + """ + # This is similar in spirit to `np.min_scalar_type` + if signed is not None: + if unsigned is not None and (not signed) is (not unsigned): + raise TypeError( + f"signed (={signed}) and unsigned (={unsigned}) keyword arguments " + "are incompatible." + ) + signed = bool(signed) + unsigned = not signed + elif unsigned is not None: + unsigned = bool(unsigned) + signed = not unsigned + + val = op.index(val) # Ensure val is integral + if val < 0: + if unsigned: + raise ValueError(f"Value is incompatible with unsigned int: {val}.") + signed = True + unsigned = False + + if signed is not False: + # Number of bytes (and a power of two) + signed_nbytes = (val + (val < 0)).bit_length() // 8 + 1 + signed_nbytes = next( + filter( + signed_nbytes.__le__, + itertools.accumulate(itertools.repeat(2), op.mul, initial=1), + ) + ) + if unsigned is not False: + # Number of bytes (and a power of two) + unsigned_nbytes = (val.bit_length() + 7) // 8 + unsigned_nbytes = next( + filter( + unsigned_nbytes.__le__, + itertools.accumulate(itertools.repeat(2), op.mul, initial=1), + ) + ) + if signed is None and unsigned is None: + # Prefer signed int if same size + signed = signed_nbytes <= unsigned_nbytes + + if signed: + dtype_string = f"i{signed_nbytes}" + else: + dtype_string = f"u{unsigned_nbytes}" + try: + return np.dtype(dtype_string) + except TypeError as exc: + raise ValueError("Value is too large to store as integer: {val}") from exc + + +def _get_float_dtype(dtype: Dtype): + """Promote dtype to float32 or float64 as appropriate.""" + if dtype is None: + return np.dtype(np.float32) + rv = np.promote_types(dtype, np.float32) + if np.float32 != rv != np.float64: + raise TypeError( + f"Dtype {dtype} cannot be safely promoted to float32 or float64" + ) + return rv diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index db3b3a22545..f309f4797a7 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "nx-cugraph" -version = "23.10.00" +dynamic = ["version"] description = "cugraph backend for NetworkX" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -32,7 +32,8 @@ classifiers = [ dependencies = [ "cupy-cuda11x>=12.0.0", "networkx>=3.0", - "pylibcugraph==23.10.*", + "numpy>=1.21", + "pylibcugraph==23.12.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.optional-dependencies] @@ -60,6 +61,9 @@ cugraph = "_nx_cugraph:get_info" [tool.setuptools] license-files = ["LICENSE"] +[tool.setuptools.dynamic] +version = {file = "nx_cugraph/VERSION"} + [tool.setuptools.packages.find] include = [ "nx_cugraph*", @@ -80,7 +84,10 @@ float_to_top = true default_section = "THIRDPARTY" known_first_party = "nx_cugraph" line_length = 88 -extend_skip_glob = ["nx_cugraph/__init__.py"] +extend_skip_glob = [ + "nx_cugraph/__init__.py", + "nx_cugraph/classes/__init__.py", +] [tool.pytest.ini_options] minversion = "6.0" @@ -109,7 +116,7 @@ addopts = [ # "-ra", # Print summary of all fails/errors "--benchmark-warmup=off", "--benchmark-max-time=0", - "--benchmark-min-rounds=3", + "--benchmark-min-rounds=1", "--benchmark-columns=min,median,max", ] @@ -156,7 +163,8 @@ ignore = [ # "SIM300", # Yoda conditions are discouraged, use ... instead (Note: we're not this picky) # "SIM401", # Use dict.get ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) # "TRY004", # Prefer `TypeError` exception for invalid type (Note: good advice, but not worth the nuisance) - # "TRY200", # Use `raise from` to specify exception cause (Note: sometimes okay to raise original exception) + "B904", # Bare `raise` inside exception clause (like TRY200; sometimes okay) + "TRY200", # Use `raise from` to specify exception cause (Note: sometimes okay to raise original exception) # Intentionally ignored "A003", # Class attribute ... is shadowing a python builtin @@ -213,6 +221,7 @@ ignore = [ # Allow assert, print, RNG, and no docstring "nx_cugraph/**/tests/*py" = ["S101", "S311", "T201", "D103", "D100"] "_nx_cugraph/__init__.py" = ["E501"] +"nx_cugraph/algorithms/**/*py" = ["D205", "D401"] # Allow flexible docstrings for algorithms [tool.ruff.flake8-annotations] mypy-init-return = true diff --git a/python/nx-cugraph/setup.py b/python/nx-cugraph/setup.py index 87c0e10646d..c4ab535923b 100644 --- a/python/nx-cugraph/setup.py +++ b/python/nx-cugraph/setup.py @@ -10,6 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup +from setuptools import find_packages, setup -setup() +packages = find_packages(include=["nx_cugraph*"]) +setup( + package_data={key: ["VERSION"] for key in packages}, +) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index b5b564e6881..057f30ef3ad 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -14,7 +14,7 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -set(pylibcugraph_version 23.10.00) +set(pylibcugraph_version 23.12.00) include(../../fetch_rapids.cmake) diff --git a/python/pylibcugraph/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/CMakeLists.txt index 6618c50122c..c2e22fc1ff7 100644 --- a/python/pylibcugraph/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/CMakeLists.txt @@ -56,6 +56,7 @@ set(cython_sources uniform_random_walks.pyx utils.pyx weakly_connected_components.pyx + replicate_edgelist.pyx ) set(linked_libraries cugraph::cugraph;cugraph::cugraph_c) diff --git a/python/pylibcugraph/pylibcugraph/VERSION b/python/pylibcugraph/pylibcugraph/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index bd47c7da184..1d02498ea30 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -87,6 +87,8 @@ from pylibcugraph.generate_rmat_edgelists import generate_rmat_edgelists +from pylibcugraph.replicate_edgelist import replicate_edgelist + from pylibcugraph.k_truss_subgraph import k_truss_subgraph from pylibcugraph.jaccard_coefficients import jaccard_coefficients @@ -98,4 +100,4 @@ from pylibcugraph import exceptions -__version__ = "23.10.00" +from pylibcugraph._version import __git_commit__, __version__ diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd index 590c5679264..28a9f5a3be5 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -51,12 +51,38 @@ cdef extern from "cugraph_c/graph.h": bool_t check, cugraph_graph_t** graph, cugraph_error_t** error) + + # Supports isolated vertices + cdef cugraph_error_code_t \ + cugraph_graph_create_sg( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* vertices, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_types, + bool_t store_transposed, + bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t check, + cugraph_graph_t** graph, + cugraph_error_t** error) # This may get renamed to cugraph_graph_free() cdef void \ cugraph_sg_graph_free( cugraph_graph_t* graph ) + + # FIXME: Might want to delete 'cugraph_sg_graph_free' and replace + # 'cugraph_mg_graph_free' by 'cugraph_graph_free' + cdef void \ + cugraph_graph_free( + cugraph_graph_t* graph + ) cdef cugraph_error_code_t \ cugraph_mg_graph_create( @@ -96,6 +122,22 @@ cdef extern from "cugraph_c/graph.h": cugraph_error_t** error ) + cdef cugraph_error_code_t \ + cugraph_graph_create_sg_from_csr( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* offsets, + const cugraph_type_erased_device_array_view_t* indices, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t check, + cugraph_graph_t** graph, + cugraph_error_t** error + ) + cdef void \ cugraph_sg_graph_free( cugraph_graph_t* graph @@ -117,6 +159,24 @@ cdef extern from "cugraph_c/graph.h": cugraph_error_t** error ) + cdef cugraph_error_code_t \ + cugraph_graph_create_mg( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t** vertices, + const cugraph_type_erased_device_array_view_t** src, + const cugraph_type_erased_device_array_view_t** dst, + const cugraph_type_erased_device_array_view_t** weights, + const cugraph_type_erased_device_array_view_t** edge_ids, + const cugraph_type_erased_device_array_view_t** edge_type_ids, + bool_t store_transposed, + size_t num_arrays, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) + cdef void \ cugraph_mg_graph_free( cugraph_graph_t* graph diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd index f18e9848182..8b3a629956c 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -138,6 +138,16 @@ cdef extern from "cugraph_c/graph_functions.h": cugraph_induced_subgraph_result_t* induced_subgraph ) + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_result_t* induced_subgraph + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_result_t* induced_subgraph + ) + cdef cugraph_type_erased_device_array_view_t* \ cugraph_induced_subgraph_get_subgraph_offsets( cugraph_induced_subgraph_result_t* induced_subgraph @@ -158,3 +168,17 @@ cdef extern from "cugraph_c/graph_functions.h": cugraph_induced_subgraph_result_t** result, cugraph_error_t** error ) + + ########################################################################### + # allgather + cdef cugraph_error_code_t \ + cugraph_allgather( + const cugraph_resource_handle_t* handle, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + cugraph_induced_subgraph_result_t** result, + cugraph_error_t** error + ) diff --git a/python/pylibcugraph/pylibcugraph/_version.py b/python/pylibcugraph/pylibcugraph/_version.py new file mode 100644 index 00000000000..5dca7e48b3f --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_version.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.resources + +# Read VERSION file from the module that is symlinked to VERSION file +# in the root of the repo at build time or copied to the moudle at +# installation. VERSION is a separate file that allows CI build-time scripts +# to update version info (including commit hashes) without modifying +# source files. +__version__ = ( + importlib.resources.files("pylibcugraph").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/pylibcugraph/pylibcugraph/analyze_clustering_edge_cut.pyx b/python/pylibcugraph/pylibcugraph/analyze_clustering_edge_cut.pyx index 60613f27a0d..3370e71f469 100644 --- a/python/pylibcugraph/pylibcugraph/analyze_clustering_edge_cut.pyx +++ b/python/pylibcugraph/pylibcugraph/analyze_clustering_edge_cut.pyx @@ -86,7 +86,7 @@ def analyze_clustering_edge_cut(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertex, cluster) = pylibcugraph.spectral_modularity_maximization( ... resource_handle, G, num_clusters=5, num_eigen_vects=2, evs_tolerance=0.00001 diff --git a/python/pylibcugraph/pylibcugraph/analyze_clustering_modularity.pyx b/python/pylibcugraph/pylibcugraph/analyze_clustering_modularity.pyx index 76ba48f52b7..2e7c1d2f649 100644 --- a/python/pylibcugraph/pylibcugraph/analyze_clustering_modularity.pyx +++ b/python/pylibcugraph/pylibcugraph/analyze_clustering_modularity.pyx @@ -87,7 +87,7 @@ def analyze_clustering_modularity(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertex, cluster) = pylibcugraph.spectral_modularity_maximization( ... resource_handle, G, num_clusters=5, num_eigen_vects=2, evs_tolerance=0.00001 diff --git a/python/pylibcugraph/pylibcugraph/analyze_clustering_ratio_cut.pyx b/python/pylibcugraph/pylibcugraph/analyze_clustering_ratio_cut.pyx index 39b317e107d..c06f870d048 100644 --- a/python/pylibcugraph/pylibcugraph/analyze_clustering_ratio_cut.pyx +++ b/python/pylibcugraph/pylibcugraph/analyze_clustering_ratio_cut.pyx @@ -86,7 +86,7 @@ def analyze_clustering_ratio_cut(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertex, cluster) = pylibcugraph.spectral_modularity_maximization( ... resource_handle, G, num_clusters=5, num_eigen_vects=2, evs_tolerance=0.00001 diff --git a/python/pylibcugraph/pylibcugraph/balanced_cut_clustering.pyx b/python/pylibcugraph/pylibcugraph/balanced_cut_clustering.pyx index 5a61f9e0dd7..a1a5c8182eb 100644 --- a/python/pylibcugraph/pylibcugraph/balanced_cut_clustering.pyx +++ b/python/pylibcugraph/pylibcugraph/balanced_cut_clustering.pyx @@ -109,7 +109,7 @@ def balanced_cut_clustering(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, clusters) = pylibcugraph.balanced_cut_clustering( ... resource_handle, G, num_clusters=5, num_eigen_vects=2, evs_tolerance=0.00001 diff --git a/python/pylibcugraph/pylibcugraph/ecg.pyx b/python/pylibcugraph/pylibcugraph/ecg.pyx index c5c1fe2eda7..4188aaa213e 100644 --- a/python/pylibcugraph/pylibcugraph/ecg.pyx +++ b/python/pylibcugraph/pylibcugraph/ecg.pyx @@ -101,7 +101,7 @@ def ecg(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, clusters) = pylibcugraph.ecg(resource_handle, G) # FIXME: Check this docstring example diff --git a/python/pylibcugraph/pylibcugraph/edge_betweenness_centrality.pyx b/python/pylibcugraph/pylibcugraph/edge_betweenness_centrality.pyx index c88c9fe8a67..e1dae1ff10a 100644 --- a/python/pylibcugraph/pylibcugraph/edge_betweenness_centrality.pyx +++ b/python/pylibcugraph/pylibcugraph/edge_betweenness_centrality.pyx @@ -180,7 +180,7 @@ def edge_betweenness_centrality(ResourceHandle resource_handle, cdef cugraph_type_erased_device_array_view_t* values_ptr = \ cugraph_edge_centrality_result_get_values(result_ptr) - if graph.edge_id_view_ptr is NULL: + if graph.edge_id_view_ptr is NULL and graph.edge_id_view_ptr_ptr is NULL: cupy_edge_ids = None else: edge_ids_ptr = cugraph_edge_centrality_result_get_edge_ids(result_ptr) diff --git a/python/pylibcugraph/pylibcugraph/egonet.pyx b/python/pylibcugraph/pylibcugraph/egonet.pyx index d011d946e46..e7237cc3ba4 100644 --- a/python/pylibcugraph/pylibcugraph/egonet.pyx +++ b/python/pylibcugraph/pylibcugraph/egonet.pyx @@ -101,7 +101,7 @@ def ego_graph(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=False, do_expensive_check=False) >>> (sources, destinations, edge_weights, subgraph_offsets) = ... pylibcugraph.ego_graph(resource_handle, G, source_vertices, 2, False) diff --git a/python/pylibcugraph/pylibcugraph/eigenvector_centrality.pyx b/python/pylibcugraph/pylibcugraph/eigenvector_centrality.pyx index 88612c242e2..568f072ee3d 100644 --- a/python/pylibcugraph/pylibcugraph/eigenvector_centrality.pyx +++ b/python/pylibcugraph/pylibcugraph/eigenvector_centrality.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -97,7 +97,7 @@ def eigenvector_centrality(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, values) = pylibcugraph.eigenvector_centrality( resource_handle, G, 1e-6, 1000, False) diff --git a/python/pylibcugraph/pylibcugraph/graphs.pxd b/python/pylibcugraph/pylibcugraph/graphs.pxd index a2df44ba26e..dac69e0ad04 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pxd +++ b/python/pylibcugraph/pylibcugraph/graphs.pxd @@ -25,11 +25,13 @@ from pylibcugraph._cugraph_c.graph cimport ( cdef class _GPUGraph: cdef cugraph_graph_t* c_graph_ptr cdef cugraph_type_erased_device_array_view_t* edge_id_view_ptr - cdef cugraph_type_erased_device_array_view_t* weights_view_ptr + cdef cugraph_type_erased_device_array_view_t** edge_id_view_ptr_ptr + cdef cugraph_type_erased_device_array_view_t* weights_view_ptr + cdef cugraph_type_erased_device_array_view_t** weights_view_ptr_ptr cdef class SGGraph(_GPUGraph): pass cdef class MGGraph(_GPUGraph): - pass + pass diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index 33a8a09c6f4..b3065fa0684 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -18,16 +18,20 @@ from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) +from cython.operator cimport dereference as deref from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_view_t, cugraph_type_erased_device_array_view_free, ) from pylibcugraph._cugraph_c.graph cimport ( - cugraph_sg_graph_create, - cugraph_mg_graph_create, - cugraph_sg_graph_create_from_csr, - cugraph_sg_graph_free, - cugraph_mg_graph_free, + cugraph_graph_create_sg, + cugraph_graph_create_mg, + cugraph_sg_graph_create_from_csr, #FIXME: Remove this once + # 'cugraph_graph_create_sg_from_csr' is exposed + cugraph_graph_create_sg_from_csr, + cugraph_sg_graph_free, #FIXME: Remove this + cugraph_graph_free, + cugraph_mg_graph_free, #FIXME: Remove this ) from pylibcugraph.resource_handle cimport ( ResourceHandle, @@ -38,8 +42,11 @@ from pylibcugraph.graph_properties cimport ( from pylibcugraph.utils cimport ( assert_success, assert_CAI_type, + get_c_type_from_numpy_type, create_cugraph_type_erased_device_array_view_from_py_obj, ) +from libc.stdlib cimport malloc + cdef class SGGraph(_GPUGraph): @@ -70,6 +77,9 @@ cdef class SGGraph(_GPUGraph): CSR format. In the case of a COO, The order of the array corresponds to the ordering of the src_offset_array, where the ith item in src_offset_array and the ith item in dst_index_array define the ith edge of the graph. + + vertices_array : device array type + Device array containing the isolated vertices of the graph. weight_array : device array type Device array containing the weight values of each directed edge. The @@ -105,6 +115,12 @@ cdef class SGGraph(_GPUGraph): COO: arrays represent src_array and dst_array CSR: arrays represent offset_array and index_array + drop_self_loops : bool, optional (default='False') + If true, drop any self loops that exist in the provided edge list. + + drop_multi_edges: bool, optional (default='False') + If true, drop any multi edges that exist in the provided edge list + Examples --------- >>> import pylibcugraph, cupy, numpy @@ -116,7 +132,7 @@ cdef class SGGraph(_GPUGraph): >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=False, do_expensive_check=False) """ @@ -131,7 +147,10 @@ cdef class SGGraph(_GPUGraph): do_expensive_check=False, edge_id_array=None, edge_type_array=None, - input_array_format="COO"): + input_array_format="COO", + vertices_array=None, + drop_self_loops=False, + drop_multi_edges=False): # FIXME: add tests for these if not(isinstance(store_transposed, (int, bool))): @@ -145,13 +164,13 @@ cdef class SGGraph(_GPUGraph): f"{type(do_expensive_check)}") assert_CAI_type(src_or_offset_array, "src_or_offset_array") assert_CAI_type(dst_or_index_array, "dst_or_index_array") + assert_CAI_type(vertices_array, "vertices_array", True) assert_CAI_type(weight_array, "weight_array", True) - if edge_id_array is not None: - assert_CAI_type(edge_id_array, "edge_id_array") - if edge_type_array is not None: - assert_CAI_type(edge_type_array, "edge_type_array") + assert_CAI_type(edge_id_array, "edge_id_array", True) + assert_CAI_type(edge_type_array, "edge_type_array", True) - # FIXME: assert that src_or_offset_array and dst_or_index_array have the same type + # FIXME: assert that src_or_offset_array and dst_or_index_array have + # the same type cdef cugraph_error_t* error_ptr cdef cugraph_error_code_t error_code @@ -159,31 +178,31 @@ cdef class SGGraph(_GPUGraph): cdef cugraph_type_erased_device_array_view_t* srcs_or_offsets_view_ptr = \ create_cugraph_type_erased_device_array_view_from_py_obj( src_or_offset_array - ) - + ) cdef cugraph_type_erased_device_array_view_t* dsts_or_indices_view_ptr = \ create_cugraph_type_erased_device_array_view_from_py_obj( dst_or_index_array ) - - + cdef cugraph_type_erased_device_array_view_t* vertices_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj( + vertices_array + ) self.weights_view_ptr = create_cugraph_type_erased_device_array_view_from_py_obj( weight_array ) - self.edge_id_view_ptr = create_cugraph_type_erased_device_array_view_from_py_obj( edge_id_array - ) - + ) cdef cugraph_type_erased_device_array_view_t* edge_type_view_ptr = \ create_cugraph_type_erased_device_array_view_from_py_obj( edge_type_array ) if input_array_format == "COO": - error_code = cugraph_sg_graph_create( + error_code = cugraph_graph_create_sg( resource_handle.c_resource_handle_ptr, &(graph_properties.c_graph_properties), + vertices_view_ptr, srcs_or_offsets_view_ptr, dsts_or_indices_view_ptr, self.weights_view_ptr, @@ -191,12 +210,13 @@ cdef class SGGraph(_GPUGraph): edge_type_view_ptr, store_transposed, renumber, + drop_self_loops, + drop_multi_edges, do_expensive_check, &(self.c_graph_ptr), &error_ptr) - assert_success(error_code, error_ptr, - "cugraph_sg_graph_create()") + "cugraph_graph_create_sg()") elif input_array_format == "CSR": error_code = cugraph_sg_graph_create_from_csr( @@ -209,6 +229,8 @@ cdef class SGGraph(_GPUGraph): edge_type_view_ptr, store_transposed, renumber, + # drop_self_loops, #FIXME: Not supported yet + # drop_multi_edges, #FIXME: Not supported yet do_expensive_check, &(self.c_graph_ptr), &error_ptr) @@ -223,7 +245,8 @@ cdef class SGGraph(_GPUGraph): cugraph_type_erased_device_array_view_free(srcs_or_offsets_view_ptr) cugraph_type_erased_device_array_view_free(dsts_or_indices_view_ptr) - cugraph_type_erased_device_array_view_free(self.weights_view_ptr) + if self.weights_view_ptr is not NULL: + cugraph_type_erased_device_array_view_free(self.weights_view_ptr) if self.edge_id_view_ptr is not NULL: cugraph_type_erased_device_array_view_free(self.edge_id_view_ptr) if edge_type_view_ptr is not NULL: @@ -259,6 +282,9 @@ cdef class MGGraph(_GPUGraph): each directed edge. The order of the array corresponds to the ordering of the src_array, where the ith item in src_array and the ith item in dst_array define the ith edge of the graph. + + vertices_array : device array type + Device array containing the isolated vertices of the graph. weight_array : device array type Device array containing the weight values of each directed edge. The @@ -270,8 +296,10 @@ cdef class MGGraph(_GPUGraph): Set to True if the graph should be transposed. This is required for some algorithms, such as pagerank. - num_edges : int - Number of edges + num_arrays : size_t + Number of arrays. + + If provided, all list of device arrays should be of the same size. do_expensive_check : bool If True, performs more extensive tests on the inputs to ensure @@ -286,6 +314,12 @@ cdef class MGGraph(_GPUGraph): Device array containing the edge types of each directed edge. Must match the ordering of the src/dst/edge_id arrays. Optional (may be null). If provided, edge_id_array must be provided. + + drop_self_loops : bool, optional (default='False') + If true, drop any self loops that exist in the provided edge list. + + drop_multi_edges: bool, optional (default='False') + If true, drop any multi edges that exist in the provided edge list """ def __cinit__(self, ResourceHandle resource_handle, @@ -294,85 +328,156 @@ cdef class MGGraph(_GPUGraph): dst_array, weight_array=None, store_transposed=False, - num_edges=-1, - do_expensive_check=False, + do_expensive_check=False, # default to False edge_id_array=None, - edge_type_array=None): + edge_type_array=None, + vertices_array=None, + size_t num_arrays=1, # default value to not break users + drop_self_loops=False, + drop_multi_edges=False): - # FIXME: add tests for these if not(isinstance(store_transposed, (int, bool))): raise TypeError("expected int or bool for store_transposed, got " f"{type(store_transposed)}") - if not(isinstance(num_edges, (int))): - raise TypeError("expected int for num_edges, got " - f"{type(num_edges)}") - if num_edges < 0: - raise TypeError("num_edges must be > 0") + if not(isinstance(do_expensive_check, (int, bool))): raise TypeError("expected int or bool for do_expensive_check, got " f"{type(do_expensive_check)}") - assert_CAI_type(src_array, "src_array") - assert_CAI_type(dst_array, "dst_array") - assert_CAI_type(weight_array, "weight_array", True) - - assert_CAI_type(edge_id_array, "edge_id_array", True) - if edge_id_array is not None and len(edge_id_array) != len(src_array): - raise ValueError('Edge id array must be same length as edgelist') - - assert_CAI_type(edge_type_array, "edge_type_array", True) - if edge_type_array is not None and len(edge_type_array) != len(src_array): - raise ValueError('Edge type array must be same length as edgelist') - - # FIXME: assert that src_array and dst_array have the same type cdef cugraph_error_t* error_ptr cdef cugraph_error_code_t error_code - cdef cugraph_type_erased_device_array_view_t* srcs_view_ptr = \ - create_cugraph_type_erased_device_array_view_from_py_obj( - src_array - ) - cdef cugraph_type_erased_device_array_view_t* dsts_view_ptr = \ - create_cugraph_type_erased_device_array_view_from_py_obj( - dst_array - ) - self.weights_view_ptr = \ - create_cugraph_type_erased_device_array_view_from_py_obj( - weight_array - ) - self.edge_id_view_ptr = \ - create_cugraph_type_erased_device_array_view_from_py_obj( - edge_id_array - ) - cdef cugraph_type_erased_device_array_view_t* edge_type_view_ptr = \ - create_cugraph_type_erased_device_array_view_from_py_obj( - edge_type_array - ) - error_code = cugraph_mg_graph_create( + if not isinstance(src_array, list): + src_array = [src_array] + if not any(src_array): + src_array = src_array * num_arrays + + if not isinstance(dst_array, list): + dst_array = [dst_array] + if not any(dst_array): + dst_array = dst_array * num_arrays + + if not isinstance(weight_array, list): + weight_array = [weight_array] + if not any(weight_array): + weight_array = weight_array * num_arrays + + if not isinstance(edge_id_array, list): + edge_id_array = [edge_id_array] + if not any(edge_id_array): + edge_id_array = edge_id_array * num_arrays + + if not isinstance(edge_type_array, list): + edge_type_array = [edge_type_array] + if not any(edge_type_array): + edge_type_array = edge_type_array * num_arrays + + if not isinstance(vertices_array, list): + vertices_array = [vertices_array] + if not any(vertices_array): + vertices_array = vertices_array * num_arrays + + cdef cugraph_type_erased_device_array_view_t** srcs_view_ptr_ptr = NULL + cdef cugraph_type_erased_device_array_view_t** dsts_view_ptr_ptr = NULL + cdef cugraph_type_erased_device_array_view_t** vertices_view_ptr_ptr = NULL + cdef cugraph_type_erased_device_array_view_t** edge_type_view_ptr_ptr = NULL + + for i in range(num_arrays): + if do_expensive_check: + assert_CAI_type(src_array[i], "src_array") + assert_CAI_type(dst_array[i], "dst_array") + assert_CAI_type(weight_array[i], "weight_array", True) + assert_CAI_type(vertices_array[i], "vertices_array", True) + + assert_CAI_type(edge_id_array[i], "edge_id_array", True) + + if edge_id_array is not None and len(edge_id_array[i]) != len(src_array[i]): + raise ValueError('Edge id array must be same length as edgelist') + + assert_CAI_type(edge_type_array[i], "edge_type_array", True) + if edge_type_array[i] is not None and len(edge_type_array[i]) != len(src_array[i]): + raise ValueError('Edge type array must be same length as edgelist') + + if src_array[i] is not None: + if i == 0: + srcs_view_ptr_ptr = \ + malloc( + num_arrays * sizeof(cugraph_type_erased_device_array_view_t*)) + srcs_view_ptr_ptr[i] = \ + create_cugraph_type_erased_device_array_view_from_py_obj(src_array[i]) + + if dst_array[i] is not None: + if i == 0: + dsts_view_ptr_ptr = \ + malloc( + num_arrays * sizeof(cugraph_type_erased_device_array_view_t*)) + dsts_view_ptr_ptr[i] = \ + create_cugraph_type_erased_device_array_view_from_py_obj(dst_array[i]) + + if vertices_array[i] is not None: + if i == 0: + vertices_view_ptr_ptr = \ + malloc( + num_arrays * sizeof(cugraph_type_erased_device_array_view_t*)) + vertices_view_ptr_ptr[i] = \ + create_cugraph_type_erased_device_array_view_from_py_obj(vertices_array[i]) + + if weight_array[i] is not None: + if i == 0: + self.weights_view_ptr_ptr = \ + malloc( + num_arrays * sizeof(cugraph_type_erased_device_array_view_t*)) + self.weights_view_ptr_ptr[i] = \ + create_cugraph_type_erased_device_array_view_from_py_obj(weight_array[i]) + + if edge_id_array[i] is not None: + if i == 0: + self.edge_id_view_ptr_ptr = \ + malloc( + num_arrays * sizeof(cugraph_type_erased_device_array_view_t*)) + self.edge_id_view_ptr_ptr[i] = \ + create_cugraph_type_erased_device_array_view_from_py_obj(edge_id_array[i]) + + if edge_type_array[i] is not None: + if i == 0: + edge_type_view_ptr_ptr = \ + malloc( + num_arrays * sizeof(cugraph_type_erased_device_array_view_t*)) + edge_type_view_ptr_ptr[i] = \ + create_cugraph_type_erased_device_array_view_from_py_obj(edge_type_array[i]) + + error_code = cugraph_graph_create_mg( resource_handle.c_resource_handle_ptr, &(graph_properties.c_graph_properties), - srcs_view_ptr, - dsts_view_ptr, - self.weights_view_ptr, - self.edge_id_view_ptr, - edge_type_view_ptr, + vertices_view_ptr_ptr, + srcs_view_ptr_ptr, + dsts_view_ptr_ptr, + self.weights_view_ptr_ptr, + self.edge_id_view_ptr_ptr, + edge_type_view_ptr_ptr, store_transposed, - num_edges, + num_arrays, do_expensive_check, + drop_self_loops, + drop_multi_edges, &(self.c_graph_ptr), &error_ptr) assert_success(error_code, error_ptr, "cugraph_mg_graph_create()") - cugraph_type_erased_device_array_view_free(srcs_view_ptr) - cugraph_type_erased_device_array_view_free(dsts_view_ptr) - cugraph_type_erased_device_array_view_free(self.weights_view_ptr) - if self.edge_id_view_ptr is not NULL: - cugraph_type_erased_device_array_view_free(self.edge_id_view_ptr) - if edge_type_view_ptr is not NULL: - cugraph_type_erased_device_array_view_free(edge_type_view_ptr) + for i in range(num_arrays): + cugraph_type_erased_device_array_view_free(srcs_view_ptr_ptr[i]) + cugraph_type_erased_device_array_view_free(dsts_view_ptr_ptr[i]) + if vertices_view_ptr_ptr is not NULL: + cugraph_type_erased_device_array_view_free(vertices_view_ptr_ptr[i]) + if self.weights_view_ptr_ptr is not NULL: + cugraph_type_erased_device_array_view_free(self.weights_view_ptr_ptr[i]) + if self.edge_id_view_ptr_ptr is not NULL: + cugraph_type_erased_device_array_view_free(self.edge_id_view_ptr_ptr[i]) + if edge_type_view_ptr_ptr is not NULL: + cugraph_type_erased_device_array_view_free(edge_type_view_ptr_ptr[i]) def __dealloc__(self): if self.c_graph_ptr is not NULL: diff --git a/python/pylibcugraph/pylibcugraph/induced_subgraph.pyx b/python/pylibcugraph/pylibcugraph/induced_subgraph.pyx index aab36d3d5e0..99b89ec2a58 100644 --- a/python/pylibcugraph/pylibcugraph/induced_subgraph.pyx +++ b/python/pylibcugraph/pylibcugraph/induced_subgraph.pyx @@ -98,7 +98,7 @@ def induced_subgraph(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=False, do_expensive_check=False) >>> (sources, destinations, edge_weights, subgraph_offsets) = ... pylibcugraph.induced_subgraph( diff --git a/python/pylibcugraph/pylibcugraph/k_truss_subgraph.pyx b/python/pylibcugraph/pylibcugraph/k_truss_subgraph.pyx index cc91e76dd55..2c22c618249 100644 --- a/python/pylibcugraph/pylibcugraph/k_truss_subgraph.pyx +++ b/python/pylibcugraph/pylibcugraph/k_truss_subgraph.pyx @@ -96,7 +96,7 @@ def k_truss_subgraph(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=False, do_expensive_check=False) >>> (sources, destinations, edge_weights, subgraph_offsets) = ... pylibcugraph.k_truss_subgraph(resource_handle, G, k, False) diff --git a/python/pylibcugraph/pylibcugraph/leiden.pyx b/python/pylibcugraph/pylibcugraph/leiden.pyx index 87286234f16..04f8887551c 100644 --- a/python/pylibcugraph/pylibcugraph/leiden.pyx +++ b/python/pylibcugraph/pylibcugraph/leiden.pyx @@ -116,7 +116,7 @@ def leiden(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, clusters, modularity) = pylibcugraph.Leiden( resource_handle, G, 100, 1., False) diff --git a/python/pylibcugraph/pylibcugraph/louvain.pyx b/python/pylibcugraph/pylibcugraph/louvain.pyx index eca569d7da1..58f4f10bc18 100644 --- a/python/pylibcugraph/pylibcugraph/louvain.pyx +++ b/python/pylibcugraph/pylibcugraph/louvain.pyx @@ -103,7 +103,7 @@ def louvain(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, clusters, modularity) = pylibcugraph.louvain( resource_handle, G, 100, 1e-7, 1., False) diff --git a/python/pylibcugraph/pylibcugraph/node2vec.pyx b/python/pylibcugraph/pylibcugraph/node2vec.pyx index d0ab3f22b00..5d83fc46c3c 100644 --- a/python/pylibcugraph/pylibcugraph/node2vec.pyx +++ b/python/pylibcugraph/pylibcugraph/node2vec.pyx @@ -115,7 +115,7 @@ def node2vec(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=False, do_expensive_check=False) >>> (paths, weights, sizes) = pylibcugraph.node2vec( ... resource_handle, G, seeds, 3, True, 1.0, 1.0) diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index f831d844338..9fec1328bbf 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -154,7 +154,7 @@ def pagerank(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, pageranks) = pylibcugraph.pagerank( ... resource_handle, G, None, None, None, None, alpha=0.85, diff --git a/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx b/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx index 79ef80be549..85addffa694 100644 --- a/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx @@ -161,7 +161,7 @@ def personalized_pagerank(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, pageranks) = pylibcugraph.personalized_pagerank( ... resource_handle, G, None, None, None, None, alpha=0.85, diff --git a/python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx b/python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx new file mode 100644 index 00000000000..3763d4bc69d --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx @@ -0,0 +1,202 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Have cython use python 3 syntax +# cython: language_level = 3 + + +from pylibcugraph._cugraph_c.resource_handle cimport ( + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c.array cimport ( + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_device_array_view_free, +) +from pylibcugraph._cugraph_c.graph_functions cimport ( + cugraph_allgather, + cugraph_induced_subgraph_result_t, + cugraph_induced_subgraph_get_sources, + cugraph_induced_subgraph_get_destinations, + cugraph_induced_subgraph_get_edge_weights, + cugraph_induced_subgraph_get_edge_ids, + cugraph_induced_subgraph_get_edge_type_ids, + cugraph_induced_subgraph_get_subgraph_offsets, + cugraph_induced_subgraph_result_free, +) +from pylibcugraph.resource_handle cimport ( + ResourceHandle, +) +from pylibcugraph.utils cimport ( + assert_success, + assert_CAI_type, + copy_to_cupy_array, + create_cugraph_type_erased_device_array_view_from_py_obj +) + + +def replicate_edgelist(ResourceHandle resource_handle, + src_array, + dst_array, + weight_array, + edge_id_array, + edge_type_id_array): + """ + Replicate edges across all GPUs + + Parameters + ---------- + resource_handle : ResourceHandle + Handle to the underlying device resources needed for referencing data + and running algorithms. + + src_array : device array type, optional + Device array containing the vertex identifiers of the source of each + directed edge. The order of the array corresponds to the ordering of the + dst_array, where the ith item in src_array and the ith item in dst_array + define the ith edge of the graph. + + dst_array : device array type, optional + Device array containing the vertex identifiers of the destination of + each directed edge. The order of the array corresponds to the ordering + of the src_array, where the ith item in src_array and the ith item in + dst_array define the ith edge of the graph. + + weight_array : device array type, optional + Device array containing the weight values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in weight_array is the weight value + of the ith edge of the graph. + + edge_id_array : device array type, optional + Device array containing the edge id values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in edge_id_array is the id value + of the ith edge of the graph. + + edge_type_id_array : device array type, optional + Device array containing the edge type id values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in edge_type_id_array is the type id + value of the ith edge of the graph. + + Returns + ------- + return cupy arrays of 'src' and/or 'dst' and/or 'weight'and/or 'edge_id' + and/or 'edge_type_id'. + """ + assert_CAI_type(src_array, "src_array", True) + assert_CAI_type(dst_array, "dst_array", True) + assert_CAI_type(weight_array, "weight_array", True) + assert_CAI_type(edge_id_array, "edge_id_array", True) + assert_CAI_type(edge_type_id_array, "edge_type_id_array", True) + cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ + resource_handle.c_resource_handle_ptr + + cdef cugraph_induced_subgraph_result_t* result_ptr + cdef cugraph_error_code_t error_code + cdef cugraph_error_t* error_ptr + + cdef cugraph_type_erased_device_array_view_t* srcs_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(src_array) + + cdef cugraph_type_erased_device_array_view_t* dsts_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(dst_array) + + + cdef cugraph_type_erased_device_array_view_t* weights_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(weight_array) + + cdef cugraph_type_erased_device_array_view_t* edge_ids_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(edge_id_array) + + cdef cugraph_type_erased_device_array_view_t* edge_type_ids_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(edge_type_id_array) + + error_code = cugraph_allgather(c_resource_handle_ptr, + srcs_view_ptr, + dsts_view_ptr, + weights_view_ptr, + edge_ids_view_ptr, + edge_type_ids_view_ptr, + &result_ptr, + &error_ptr) + assert_success(error_code, error_ptr, "replicate_edgelist") + # Extract individual device array pointers from result and copy to cupy + # arrays for returning. + cdef cugraph_type_erased_device_array_view_t* sources_ptr + if src_array is not None: + sources_ptr = cugraph_induced_subgraph_get_sources(result_ptr) + cdef cugraph_type_erased_device_array_view_t* destinations_ptr + if dst_array is not None: + destinations_ptr = cugraph_induced_subgraph_get_destinations(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_weights_ptr = \ + cugraph_induced_subgraph_get_edge_weights(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_ids_ptr = \ + cugraph_induced_subgraph_get_edge_ids(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_type_ids_ptr = \ + cugraph_induced_subgraph_get_edge_type_ids(result_ptr) + cdef cugraph_type_erased_device_array_view_t* subgraph_offsets_ptr = \ + cugraph_induced_subgraph_get_subgraph_offsets(result_ptr) + + # FIXME: Get ownership of the result data instead of performing a copy + # for perfomance improvement + + cupy_sources = None + cupy_destinations = None + cupy_edge_weights = None + cupy_edge_ids = None + cupy_edge_type_ids = None + + if src_array is not None: + cupy_sources = copy_to_cupy_array( + c_resource_handle_ptr, sources_ptr) + + if dst_array is not None: + cupy_destinations = copy_to_cupy_array( + c_resource_handle_ptr, destinations_ptr) + + if weight_array is not None: + cupy_edge_weights = copy_to_cupy_array( + c_resource_handle_ptr, edge_weights_ptr) + + if edge_id_array is not None: + cupy_edge_ids = copy_to_cupy_array( + c_resource_handle_ptr, edge_ids_ptr) + + if edge_type_id_array is not None: + cupy_edge_type_ids = copy_to_cupy_array( + c_resource_handle_ptr, edge_type_ids_ptr) + + cupy_subgraph_offsets = copy_to_cupy_array( + c_resource_handle_ptr, subgraph_offsets_ptr) + + # Free pointer + cugraph_induced_subgraph_result_free(result_ptr) + if src_array is not None: + cugraph_type_erased_device_array_view_free(srcs_view_ptr) + if dst_array is not None: + cugraph_type_erased_device_array_view_free(dsts_view_ptr) + if weight_array is not None: + cugraph_type_erased_device_array_view_free(weights_view_ptr) + if edge_id_array is not None: + cugraph_type_erased_device_array_view_free(edge_ids_view_ptr) + if edge_type_id_array is not None: + cugraph_type_erased_device_array_view_free(edge_type_ids_view_ptr) + + return (cupy_sources, cupy_destinations, + cupy_edge_weights, cupy_edge_ids, + cupy_edge_type_ids, cupy_subgraph_offsets) diff --git a/python/pylibcugraph/pylibcugraph/spectral_modularity_maximization.pyx b/python/pylibcugraph/pylibcugraph/spectral_modularity_maximization.pyx index c74b1f0db41..fa01714744d 100644 --- a/python/pylibcugraph/pylibcugraph/spectral_modularity_maximization.pyx +++ b/python/pylibcugraph/pylibcugraph/spectral_modularity_maximization.pyx @@ -109,7 +109,7 @@ def spectral_modularity_maximization(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, clusters) = pylibcugraph.spectral_modularity_maximization( ... resource_handle, G, num_clusters=5, num_eigen_vects=2, evs_tolerance=0.00001 diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index b2cd829cb2e..56765c4a1b8 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -109,7 +109,7 @@ def sssp(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=False, do_expensive_check=False) >>> (vertices, distances, predecessors) = pylibcugraph.sssp( ... resource_handle, G, source=1, cutoff=999, diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index a7fcbfdb42a..228147a6e9f 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -135,11 +135,11 @@ def create_SGGraph(device_srcs, device_dsts, device_weights, transposed=False): graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) g = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=transposed, renumber=False, do_expensive_check=False, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_eigenvector_centrality.py b/python/pylibcugraph/pylibcugraph/tests/test_eigenvector_centrality.py index b4ff29f31c4..551dd58bdd6 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_eigenvector_centrality.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_eigenvector_centrality.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -56,11 +56,11 @@ def _generic_eigenvector_test( resource_handle = ResourceHandle() graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) G = SGGraph( - resource_handle, - graph_props, - src_arr, - dst_arr, - wgt_arr, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=src_arr, + dst_or_index_array=dst_arr, + weight_array=wgt_arr, store_transposed=False, renumber=False, do_expensive_check=True, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py index 4ebb6f1895e..b555a9a16bb 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py @@ -85,11 +85,11 @@ def test_sg_graph(graph_data): if is_valid: g = SGGraph( # noqa:F841 - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=False, renumber=False, do_expensive_check=False, @@ -100,11 +100,11 @@ def test_sg_graph(graph_data): else: with pytest.raises(ValueError): SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=False, renumber=False, do_expensive_check=False, @@ -130,7 +130,6 @@ def test_SGGraph_create_from_cudf(): SGGraph, ) - print("get edgelist...", end="", flush=True) edgelist = cudf.DataFrame( { "src": [0, 1, 2], @@ -139,10 +138,6 @@ def test_SGGraph_create_from_cudf(): } ) - print("edgelist = ", edgelist) - print("done", flush=True) - print("create Graph...", end="", flush=True) - graph_props = GraphProperties(is_multigraph=False, is_symmetric=False) plc_graph = SGGraph( diff --git a/python/pylibcugraph/pylibcugraph/tests/test_katz_centrality.py b/python/pylibcugraph/pylibcugraph/tests/test_katz_centrality.py index d12f90426fa..9550d3be481 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_katz_centrality.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_katz_centrality.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -53,11 +53,11 @@ def _generic_katz_test( resource_handle = ResourceHandle() graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) G = SGGraph( - resource_handle, - graph_props, - src_arr, - dst_arr, - wgt_arr, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=src_arr, + dst_or_index_array=dst_arr, + weight_array=wgt_arr, store_transposed=False, renumber=False, do_expensive_check=True, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_louvain.py b/python/pylibcugraph/pylibcugraph/tests/test_louvain.py index adea5e01f15..620c50f8412 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_louvain.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_louvain.py @@ -81,11 +81,11 @@ def test_sg_louvain_cupy(): resolution = 1.0 sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=False, renumber=True, do_expensive_check=False, @@ -135,11 +135,11 @@ def test_sg_louvain_cudf(): resolution = 1.0 sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=False, renumber=True, do_expensive_check=False, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_node2vec.py b/python/pylibcugraph/pylibcugraph/tests/test_node2vec.py index 0e400a5306c..fb303ce8047 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_node2vec.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_node2vec.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -94,11 +94,11 @@ def _run_node2vec( resource_handle = ResourceHandle() graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) G = SGGraph( - resource_handle, - graph_props, - src_arr, - dst_arr, - wgt_arr, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=src_arr, + dst_or_index_array=dst_arr, + weight_array=wgt_arr, store_transposed=False, renumber=renumbered, do_expensive_check=True, @@ -795,11 +795,11 @@ def test_node2vec_renumber_cupy(graph_file, renumber): resource_handle = ResourceHandle() graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) G = SGGraph( - resource_handle, - graph_props, - src_arr, - dst_arr, - wgt_arr, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=src_arr, + dst_or_index_array=dst_arr, + weight_array=wgt_arr, store_transposed=False, renumber=renumber, do_expensive_check=True, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index 56c4878324f..2a313a33f83 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py index ab46af4ff55..6ffbab76ae2 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/tests/test_triangle_count.py b/python/pylibcugraph/pylibcugraph/tests/test_triangle_count.py index aa0d5cd35f5..1862f94ac26 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_triangle_count.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_triangle_count.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -81,11 +81,11 @@ def test_sg_triangle_count_cupy(): start_list = None sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=False, renumber=True, do_expensive_check=False, @@ -131,11 +131,11 @@ def test_sg_triangle_count_cudf(): start_list = None sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=False, renumber=True, do_expensive_check=False, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_uniform_neighbor_sample.py b/python/pylibcugraph/pylibcugraph/tests/test_uniform_neighbor_sample.py index ac04635edcf..ffa90731483 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_uniform_neighbor_sample.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_uniform_neighbor_sample.py @@ -95,11 +95,11 @@ def test_neighborhood_sampling_cupy( num_edges = max(len(device_srcs), len(device_dsts)) sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=store_transposed, renumber=renumber, do_expensive_check=False, @@ -153,11 +153,11 @@ def test_neighborhood_sampling_cudf( num_edges = max(len(device_srcs), len(device_dsts)) sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=store_transposed, renumber=renumber, do_expensive_check=False, @@ -203,11 +203,11 @@ def test_neighborhood_sampling_large_sg_graph(gpubenchmark): fanout_vals = np.asarray([1, 2], dtype=np.int32) sg = SGGraph( - resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, + resource_handle=resource_handle, + graph_properties=graph_props, + src_or_offset_array=device_srcs, + dst_or_index_array=device_dsts, + weight_array=device_weights, store_transposed=True, renumber=False, do_expensive_check=False, diff --git a/python/pylibcugraph/pylibcugraph/tests/test_utils.py b/python/pylibcugraph/pylibcugraph/tests/test_utils.py index 036a62b9c1e..64947c21b74 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_utils.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/uniform_random_walks.pyx b/python/pylibcugraph/pylibcugraph/uniform_random_walks.pyx index 1570523beb8..677695f93a9 100644 --- a/python/pylibcugraph/pylibcugraph/uniform_random_walks.pyx +++ b/python/pylibcugraph/pylibcugraph/uniform_random_walks.pyx @@ -116,7 +116,7 @@ def uniform_random_walks(ResourceHandle resource_handle, cdef cugraph_type_erased_device_array_view_t* path_ptr = \ cugraph_random_walk_result_get_paths(result_ptr) - if input_graph.weights_view_ptr is NULL: + if input_graph.weights_view_ptr is NULL and input_graph.weights_view_ptr_ptr is NULL: cupy_weights = None else: weights_ptr = cugraph_random_walk_result_get_weights(result_ptr) diff --git a/python/pylibcugraph/pylibcugraph/weakly_connected_components.pyx b/python/pylibcugraph/pylibcugraph/weakly_connected_components.pyx index 7cc0d8ab4c1..240c374353d 100644 --- a/python/pylibcugraph/pylibcugraph/weakly_connected_components.pyx +++ b/python/pylibcugraph/pylibcugraph/weakly_connected_components.pyx @@ -129,7 +129,7 @@ def weakly_connected_components(ResourceHandle resource_handle, >>> graph_props = pylibcugraph.GraphProperties( ... is_symmetric=True, is_multigraph=False) >>> G = pylibcugraph.SGGraph( - ... resource_handle, graph_props, srcs, dsts, weights, + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, ... store_transposed=False, renumber=True, do_expensive_check=False) >>> (vertices, labels) = weakly_connected_components( ... resource_handle, G, None, None, None, None, False) diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 806ea65ac6c..96f5ec84efb 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -6,8 +6,8 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", "ninja", - "pylibraft==23.10.*", - "rmm==23.10.*", + "pylibraft==23.12.*", + "rmm==23.12.*", "scikit-build>=0.13.1", "setuptools>=61.0.0", "wheel", @@ -19,7 +19,7 @@ testpaths = ["pylibcugraph/tests"] [project] name = "pylibcugraph" -version = "23.10.00" +dynamic = ["version"] description = "pylibcugraph - Python bindings for the libcugraph cuGraph C/C++/CUDA library" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -28,8 +28,8 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "pylibraft==23.10.*", - "rmm==23.10.*", + "pylibraft==23.12.*", + "rmm==23.12.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -40,7 +40,7 @@ classifiers = [ [project.optional-dependencies] test = [ - "cudf==23.10.*", + "cudf==23.12.*", "numpy>=1.21", "pandas", "pytest", @@ -56,3 +56,6 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [tool.setuptools] license-files = ["LICENSE"] + +[tool.setuptools.dynamic] +version = {file = "pylibcugraph/VERSION"} diff --git a/python/pylibcugraph/setup.py b/python/pylibcugraph/setup.py index f1a419f31bb..a6c1bda3b5b 100644 --- a/python/pylibcugraph/setup.py +++ b/python/pylibcugraph/setup.py @@ -54,7 +54,7 @@ def exclude_libcxx_symlink(cmake_manifest): packages = find_packages(include=["pylibcugraph*"]) setup( packages=packages, - package_data={key: ["*.pxd"] for key in packages}, + package_data={key: ["VERSION", "*.pxd"] for key in packages}, cmake_process_manifest_hook=exclude_libcxx_symlink, cmdclass={"clean": CleanCommand}, zip_safe=False, diff --git a/readme_pages/CONTRIBUTING.md b/readme_pages/CONTRIBUTING.md index 4b736b25155..ffe1ef1831b 100644 --- a/readme_pages/CONTRIBUTING.md +++ b/readme_pages/CONTRIBUTING.md @@ -68,7 +68,7 @@ If you need more context on a particular issue, please ask. # So you want to contribute code **TL;DR General Development Process** -1. Read the documentation on [building from source](./SOURCEBUILD.md) to learn how to setup, and validate, the development environment +1. Read the documentation on [building from source](../docs/cugraph/source/installation/source_build.md) to learn how to setup, and validate, the development environment 2. Read the RAPIDS [Code of Conduct](https://docs.rapids.ai/resources/conduct/) 3. Find or submit an issue to work on (include a comment that you are working issue) 4. Fork the cuGraph [repo](#fork) and Code (make sure to add unit tests)! @@ -99,7 +99,7 @@ The RAPIDS cuGraph repo cannot directly be modified. Contributions must come in ```git clone https://github.com//cugraph.git``` -Read the section on [building cuGraph from source](./SOURCEBUILD.md) to validate that the environment is correct. +Read the section on [building cuGraph from source](../docs/cugraph/source/installation/source_build.md) to validate that the environment is correct. **Pro Tip** add an upstream remote repository so that you can keep your forked repo in sync ```git remote add upstream https://github.com/rapidsai/cugraph.git```