From 980ee4b256daede373e091de281ceda9b667c4c7 Mon Sep 17 00:00:00 2001 From: Kevin Scott <151596+thekevinscott@users.noreply.github.com> Date: Sun, 8 Oct 2023 13:38:20 -0400 Subject: [PATCH] DVC clean up (#1180) --- .dockerignore | 1 - .dvcignore | 6 -- .github/actions/setup-dvc/action.yml | 59 ------------------- .github/workflows/docs.yml | 5 -- .github/workflows/tests.yml | 38 ++---------- .../src/lib/package-scripts/build-model.ts | 2 +- models/default-model/.npmignore | 2 +- models/esrgan-legacy/.npmignore | 2 +- models/esrgan-medium/.npmignore | 2 +- models/esrgan-slim/.npmignore | 2 +- models/esrgan-thick/.npmignore | 3 +- models/pixel-upsampler/.npmignore | 3 +- netlify.toml | 3 +- scripts/package-scripts/build-model.ts | 2 +- .../create-new-model-folder.ts | 1 - test/lib/node/prepare.ts | 2 +- 16 files changed, 16 insertions(+), 117 deletions(-) delete mode 100644 .dvcignore delete mode 100644 .github/actions/setup-dvc/action.yml diff --git a/.dockerignore b/.dockerignore index 66c4ddb5b..1b6d710a5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -.dvc .pnpm-store assets coverage diff --git a/.dvcignore b/.dvcignore deleted file mode 100644 index babb06c21..000000000 --- a/.dvcignore +++ /dev/null @@ -1,6 +0,0 @@ -# Add patterns of files dvc should ignore, which could improve -# the performance. Learn more at -# https://dvc.org/doc/user-guide/dvcignore -# -.DS_Store -node_modules diff --git a/.github/actions/setup-dvc/action.yml b/.github/actions/setup-dvc/action.yml deleted file mode 100644 index 142628ccc..000000000 --- a/.github/actions/setup-dvc/action.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Setup DVC -description: "Sets up DVC" - -inputs: - gdrive_credentials_data: # id of input - description: 'Path to GDrive Credentials Data' - required: true - -runs: - using: "composite" - steps: - - name: 'Setup DVC' - shell: bash - # run: pip install dvc[s3]==3.14.0 dvc[gdrive]==3.14.0 - run: pip install dvc[s3]=='2.45.1' dvc[gdrive]=='2.45.1' - # run: pip install dvc[all]=='3.2.1' - # uses: iterative/setup-dvc@v1 - # with: - # version: '2.45.1' - - - name: 'DVC Version' - shell: bash - run: dvc --version - - - name: 'DVC Doctor' - shell: bash - run: dvc doctor - - - name: Restore cached output - id: cache-output-restore - uses: actions/cache/restore@v3 - with: - path: dvc-cache-dir - key: dvc-cache-dir - - - name: 'Set Cache Dir' - shell: bash - run: dvc cache dir dvc-cache-dir - - - name: 'Set Cache Shared to Group' - shell: bash - run: dvc config cache.shared group - - - name: 'Set Cache Type to Symlink' - shell: bash - run: dvc config cache.type hardlink,symlink - - - name: 'Checkout Models' - shell: bash - run : dvc pull -vv -r gdrive-service-account - env: - GDRIVE_CREDENTIALS_DATA : ${{ inputs.gdrive_credentials_data }} - - - name: Save Cache - id: cache-output-save - uses: actions/cache/save@v3 - with: - path: dvc-cache-dir - key: ${{ steps.cache-output-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 743e68e44..18d24db05 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -47,11 +47,6 @@ jobs: # with: # detached: true - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - - name: 'Setup PNPM' uses: ./.github/actions/setup-pnpm diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ae74c009e..d0ecec36d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,10 +14,10 @@ jobs: with: lfs: 'true' - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - detached: true + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # detached: true - name: 'Setup PNPM with Node 16' uses: ./.github/actions/setup-pnpm @@ -153,11 +153,6 @@ jobs: with: node-version: 16 - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - - name: 'Build Models' run: pnpm test:integration:browserstack -- --ci --verbose --skipTest --skipUpscalerBuild --skipBundle @@ -208,11 +203,6 @@ jobs: with: node-version: 16 - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - - name: 'Build Models' run: pnpm test:integration:node -- --ci --verbose --skipTest --skipUpscalerBuild --skipBundle @@ -239,11 +229,6 @@ jobs: with: node-version: 16 - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 # with: @@ -272,11 +257,6 @@ jobs: with: node-version: 16 - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - - name: 'Build Models' run: pnpm test:model -- --ci --verbose --skipTest --skipUpscalerBuild --skipBundle @@ -300,11 +280,6 @@ jobs: with: node-version: 16 - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - - name: 'Build Models' run: pnpm test:memory-leaks -- --ci --verbose --skipTest --skipUpscalerBuild --skipBundle @@ -333,11 +308,6 @@ jobs: # with: # detached: true - # - name: 'Setup DVC & Pull Models' - # uses: ./.github/actions/setup-dvc - # with: - # gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} - - name: 'Install dependencies' run: pnpm install diff --git a/internals/upscaler-cli/src/lib/package-scripts/build-model.ts b/internals/upscaler-cli/src/lib/package-scripts/build-model.ts index 8d2ca2a54..f97227e8d 100644 --- a/internals/upscaler-cli/src/lib/package-scripts/build-model.ts +++ b/internals/upscaler-cli/src/lib/package-scripts/build-model.ts @@ -215,7 +215,7 @@ const buildModel = async ( const modelsFolder = path.resolve(MODELS_DIR, model, 'models'); const modelFiles = fs.readdirSync(modelsFolder); if (modelFiles.length === 0) { - throw new Error(`No model files found in folder ${modelsFolder}. Did you call dvc pull for ${model}?`); + throw new Error(`No model files found in folder ${modelsFolder}. Did you call or set up git lfs for ${model}?`); } } if (opts.verbose) { diff --git a/models/default-model/.npmignore b/models/default-model/.npmignore index 9d1203f00..0888bbea1 100644 --- a/models/default-model/.npmignore +++ b/models/default-model/.npmignore @@ -5,4 +5,4 @@ node_modules test demo DOC.mdx -models.dvc +.gitattributes diff --git a/models/esrgan-legacy/.npmignore b/models/esrgan-legacy/.npmignore index 9d1203f00..0888bbea1 100755 --- a/models/esrgan-legacy/.npmignore +++ b/models/esrgan-legacy/.npmignore @@ -5,4 +5,4 @@ node_modules test demo DOC.mdx -models.dvc +.gitattributes diff --git a/models/esrgan-medium/.npmignore b/models/esrgan-medium/.npmignore index 8e626a039..4f57e212c 100644 --- a/models/esrgan-medium/.npmignore +++ b/models/esrgan-medium/.npmignore @@ -5,5 +5,5 @@ node_modules test demo DOC.mdx -models.dvc doc-snippets +.gitattributes diff --git a/models/esrgan-slim/.npmignore b/models/esrgan-slim/.npmignore index 8e626a039..4f57e212c 100644 --- a/models/esrgan-slim/.npmignore +++ b/models/esrgan-slim/.npmignore @@ -5,5 +5,5 @@ node_modules test demo DOC.mdx -models.dvc doc-snippets +.gitattributes diff --git a/models/esrgan-thick/.npmignore b/models/esrgan-thick/.npmignore index 9d1203f00..4f57e212c 100644 --- a/models/esrgan-thick/.npmignore +++ b/models/esrgan-thick/.npmignore @@ -5,4 +5,5 @@ node_modules test demo DOC.mdx -models.dvc +doc-snippets +.gitattributes diff --git a/models/pixel-upsampler/.npmignore b/models/pixel-upsampler/.npmignore index 9d1203f00..4f57e212c 100644 --- a/models/pixel-upsampler/.npmignore +++ b/models/pixel-upsampler/.npmignore @@ -5,4 +5,5 @@ node_modules test demo DOC.mdx -models.dvc +doc-snippets +.gitattributes diff --git a/netlify.toml b/netlify.toml index 5c7a3c5dd..7ae46b78a 100644 --- a/netlify.toml +++ b/netlify.toml @@ -2,8 +2,7 @@ # NPM_FLAGS = "--version" # [build] # base = "docs/" -# # command = "pip install --upgrade pip && pip install dvc[all] && echo '>>>>>>>>' && GDRIVE_CREDENTIALS_DATA=$GDRIVE_CREDENTIALS_DATA dvc pull -v -r gdrive-service-account && npx pnpm i --store=node_modules/.pnpm-store && npx pnpm run build" -# command = "pip install --upgrade pip && pip install dvc[all] && echo '>>>>>>>>' && GDRIVE_CREDENTIALS_DATA=$GDRIVE_CREDENTIALS_DATA dvc pull -v -r gdrive-service-account && pnpm i --store=node_modules/.pnpm-store && pnpm run build" +# command = "pip install --upgrade pip && echo '>>>>>>>>' && pnpm i --store=node_modules/.pnpm-store && pnpm run build" # # command = "npx pnpm i --store=node_modules/.pnpm-store && npx pnpm run build" # # publish = "build" # # ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ." diff --git a/scripts/package-scripts/build-model.ts b/scripts/package-scripts/build-model.ts index 67de2f200..cd45fa57f 100644 --- a/scripts/package-scripts/build-model.ts +++ b/scripts/package-scripts/build-model.ts @@ -216,7 +216,7 @@ const buildModel = async ( const modelsFolder = path.resolve(MODELS_DIR, model, 'models'); const modelFiles = fs.readdirSync(modelsFolder); if (modelFiles.length === 0) { - throw new Error(`No model files found in folder ${modelsFolder}. Did you call dvc pull for ${model}?`); + throw new Error(`No model files found in folder ${modelsFolder}. Did you enable Git LFS for ${model}?`); } } if (opts.verbose) { diff --git a/scripts/package-scripts/create-new-model-folder.ts b/scripts/package-scripts/create-new-model-folder.ts index 1224ba240..edd7bc7b5 100644 --- a/scripts/package-scripts/create-new-model-folder.ts +++ b/scripts/package-scripts/create-new-model-folder.ts @@ -78,7 +78,6 @@ const createNpmIgnore = (): string => [ 'test', 'demo', 'DOC.mdx', - 'models.dvc', ].join('\n'); const createGitIgnore = () => [ diff --git a/test/lib/node/prepare.ts b/test/lib/node/prepare.ts index fe9f4f5c1..c8d1da42a 100644 --- a/test/lib/node/prepare.ts +++ b/test/lib/node/prepare.ts @@ -58,7 +58,7 @@ export const prepareScriptBundleForNodeCJS: Bundle = async ({ const modelsFolder = path.resolve(MODELS_DIR, packageName, 'models'); const modelFiles = fs.readdirSync(modelsFolder); if (modelFiles.length === 0) { - throw new Error(`No model files found in folder ${modelsFolder}. Did you call dvc pull for ${packageName}?`); + throw new Error(`No model files found in folder ${modelsFolder}. Did you enable Git LFS for ${packageName}?`); } return { src: path.resolve(MODELS_DIR, packageName),