diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6a74bc5..8d27a693 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,7 +82,6 @@ jobs: --file /opt/conda-libmamba-solver-src/dev/requirements.txt \ --file /opt/conda-libmamba-solver-src/tests/requirements.txt && sudo /opt/conda/bin/python -m pip install /opt/conda-libmamba-solver-src --no-deps -vvv && - sudo /opt/conda/bin/python -m pip install /opt/conda-libmamba-solver-src/dev/collect_upstream_conda_tests/ -vvv && source /opt/conda-src/dev/linux/bashrc.sh && /opt/conda/bin/python -m pytest /opt/conda-libmamba-solver-src -vv -m 'not slow'" @@ -167,7 +166,6 @@ jobs: --file ../conda-libmamba-solver/tests/requirements.txt \ python=${{ matrix.python-version }} conda update openssl ca-certificates certifi - python -m pip install ../conda-libmamba-solver/dev/collect_upstream_conda_tests -vv conda info python -c "from importlib.metadata import version; print('libmambapy', version('libmambapy'))" @@ -263,8 +261,6 @@ jobs: run: | call .\dev-init.bat if errorlevel 1 exit 1 - python -m pip install -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver\dev\collect_upstream_conda_tests" - if errorlevel 1 exit 1 python -m pip install --no-deps -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver" if errorlevel 1 exit 1 diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml index 4e6fae8a..c4f84b01 100644 --- a/.github/workflows/upstream_tests.yml +++ b/.github/workflows/upstream_tests.yml @@ -4,16 +4,18 @@ name: Upstream tests # CONDA-LIBMAMBA-SOLVER CHANGE on: - # NOTE: github.event context is push payload: - # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#push push: branches: - main - feature/** + - '[0-9].*.x' # e.g., 4.14.x + - '[0-9][0-9].*.x' # e.g., 23.3.x - # NOTE: github.event context is pull_request payload: - # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request pull_request: + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch + workflow_dispatch: # CONDA-LIBMAMBA-SOLVER CHANGE schedule: - cron: "15 7 * * 1-5" # Mon to Fri, 7:15am @@ -28,25 +30,22 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true -env: - # see https://github.com/conda/conda-libmamba-solver/pull/159 - MINIO_RELEASE: 'archive/minio.RELEASE.2023-03-13T19-46-17Z' - jobs: # detect whether any code changes are included in this PR changes: runs-on: ubuntu-latest permissions: + # necessary to detect changes + # https://github.com/dorny/paths-filter#supported-workflows pull-requests: read outputs: code: ${{ steps.filter.outputs.code }} steps: - uses: actions/checkout@v3 # dorny/paths-filter needs git clone for push events - # https://github.com/marketplace/actions/paths-changes-filter#supported-workflows - # CONDA-LIBMAMBA-SOLVER CHANGE + # https://github.com/dorny/paths-filter#supported-workflows if: github.event_name != 'pull_request' - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 + - uses: dorny/paths-filter@v2.11.1 id: filter with: # CONDA-LIBMAMBA-SOLVER CHANGE @@ -58,6 +57,7 @@ jobs: - '*.py' - 'recipe/**' - '.github/workflows/upstream_tests.yml' + - 'dev/**' # /CONDA-LIBMAMBA-SOLVER CHANGE # windows test suite @@ -81,7 +81,6 @@ jobs: env: OS: Windows PYTHON: ${{ matrix.python-version }} - CONDA_SUBDIR: ${{ matrix.conda-subdir }} TEST_SPLITS: 3 TEST_GROUP: ${{ matrix.test-group }} steps: @@ -122,16 +121,6 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-py${{ env.PYTHON }}-${{ matrix.default-channel }}-${{ hashFiles('recipe/meta.yaml', 'dev/windows/setup.bat', 'tests/requirements.txt') }} - - name: Cache minio - uses: actions/cache@v3 - env: - # Increase this value to reset cache - CACHE_NUMBER: 3 - with: - path: minio.exe - key: - ${{ runner.os }}-${{ env.MINIO_RELEASE && env.MINIO_RELEASE || 'minio' }}-${{ env.CACHE_NUMBER }} - - name: Set temp dirs correctly # https://github.com/actions/virtual-environments/issues/712 run: | @@ -156,9 +145,6 @@ jobs: call .\dev-init.bat if errorlevel 1 exit 1 :: /original logic - :: Install test collection plugin - python -m pip install -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver\dev\collect_upstream_conda_tests" - if errorlevel 1 exit 1 :: Install conda-libmamba-solver python -m pip install --no-deps -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver" if errorlevel 1 exit 1 @@ -168,30 +154,27 @@ jobs: if errorlevel 1 exit 1 # /CONDA-LIBMAMBA-SOLVER CHANGE - - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.conda-subdir }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} + - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE shell: cmd - env: - CONDA_SOLVER: libmamba # CONDA-LIBMAMBA-SOLVER CHANGE run: | call .\dev\windows\${{ matrix.test-type }}.bat - uses: codecov/codecov-action@v3 with: - flags: ${{ matrix.test-type }} - env_vars: OS,PYTHON + flags: ${{ matrix.test-type }},${{ runner.os }},${{ matrix.python-version }} - name: Upload test results - if: always() + if: '!cancelled()' uses: actions/upload-artifact@v3 with: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. - name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.conda-subdir }}-${{ matrix.test-type }}-${{ matrix.test-group }} + name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths path: | conda/.coverage - conda/.test_durations_${OS} + conda/tools/durations/${{ runner.os }}.json conda/test-report.xml retention-days: 1 @@ -239,108 +222,32 @@ jobs: # /CONDA-LIBMAMBA-SOLVER CHANGE - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} - env: - CONDA_SOLVER: libmamba # CONDA-LIBMAMBA-SOLVER CHANGE; - # we also added '-e CONDA_SOLVER' to the docker options below - # changes the paths to the volume(s) (plural, we also need conda-libmamba-solver) - # and changed the script being run to our vendored copy (last line) + # CONDA-LIBMAMBA-SOLVER CHANGES: + # - changed the paths to the volume(s) (plural, we also need conda-libmamba-solver) + # - changed the script being run to our vendored copy (last line) run: > docker run --rm -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src -e TEST_SPLITS -e TEST_GROUP - -e CONDA_SOLVER ghcr.io/conda/conda-ci:main-linux-python${{ matrix.python-version }}${{ matrix.default-channel == 'conda-forge' && '-conda-forge' || '' }} /opt/conda-libmamba-solver-src/dev/linux/upstream_${{ matrix.test-type }}.sh - uses: codecov/codecov-action@v3 with: - flags: ${{ matrix.test-type }} - env_vars: OS,PYTHON + flags: ${{ matrix.test-type }},${{ runner.os }},${{ matrix.python-version }} - name: Upload test results - if: always() + if: '!cancelled()' uses: actions/upload-artifact@v3 with: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths - path: | - conda/.coverage - conda/.test_durations_${OS} - conda/test-report.xml - retention-days: 1 - - # linux-qemu test suite - linux-qemu: - # only run test suite if there are code changes - needs: changes - if: false # needs.changes.outputs.code == 'true' # CONDA-LIBMAMBA-SOLVER CHANGE - - # Run one single fast test per docker+qemu emulated linux platform to test that - # test execution is possible there (container+tools+dependencies work). Can be - # changed / extended to run specific tests in case there are platform related - # things to test. Running more tests is time consuming due to emulation - # (factor 2-10x slower). - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - default-channel: ['defaults', 'conda-forge'] - python-version: ['3.11'] - platform: ['arm64', 'ppc64le'] - env: - OS: linux-${{ matrix.platform }} - PYTHON: ${{ matrix.python-version }} - steps: - - name: Checkout conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE - uses: actions/checkout@v3 - with: - fetch-depth: 0 - repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE - path: conda # CONDA-LIBMAMBA-SOLVER CHANGE - - # CONDA-LIBMAMBA-SOLVER CHANGE - - name: Checkout conda-libmamba-solver - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: conda-libmamba-solver - # /CONDA-LIBMAMBA-SOLVER CHANGE - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - # Equivalent to locally run: - # `docker run --privileged --rm tonistiigi/binfmt --install all` - - # CONDA-LIBMAMBA-SOLVER CHANGE - # - export CONDA_SOLVER - # - - - name: Python linux-${{ matrix.platform }} on ${{ matrix.default-channel }}, ${{ matrix.python-version }} tests - run: > - docker run - --rm - -v ${PWD}:/opt/conda-src - -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src - --platform linux/${{ matrix.platform }} - -e TEST_SPLITS - -e TEST_GROUP - ghcr.io/conda/conda-ci:main-linux-python${{ matrix.python-version }}${{ matrix.default-channel == 'conda-forge' && '-conda-forge' || '' }} - bash -c "source /opt/conda/etc/profile.d/conda.sh; \ - pytest --cov=conda -k test_DepsModifier_contract" - - - name: Upload test results - if: always() - uses: actions/upload-artifact@v3 - with: - # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate - # when locally dowloading and comparing results of different workflow runs. - name: test-results-${{ github.sha }}-linux-${{ matrix.platform }}-qemu-${{ matrix.default-channel }}-${{ matrix.python-version }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths path: | conda/.coverage + conda/tools/durations/${{ runner.os }}.json conda/test-report.xml retention-days: 1 @@ -404,16 +311,6 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-py${{ env.PYTHON }}-${{ matrix.default-channel }}-${{ hashFiles('recipe/meta.yaml', 'dev/macos/setup.sh', 'tests/requirements.txt') }} - - name: Cache minio - uses: actions/cache@v3 - env: - # Increase this value to reset cache - CACHE_NUMBER: 3 - with: - path: minio - key: - ${{ runner.os }}-${{ env.MINIO_RELEASE && env.MINIO_RELEASE || 'minio' }}-${{ env.CACHE_NUMBER }} - - uses: conda-incubator/setup-miniconda@v2 name: Setup miniconda for defaults if: matrix.default-channel == 'defaults' @@ -445,7 +342,6 @@ jobs: ./dev/macos/setup.sh # /original setup python -c "from importlib.metadata import version; print('libmambapy', version('libmambapy'))" - python -m pip install ../conda-libmamba-solver/dev/collect_upstream_conda_tests -vv python -m pip install ../conda-libmamba-solver -vv --no-deps conda info -a # /CONDA-LIBMAMBA-SOLVER CHANGE @@ -453,35 +349,31 @@ jobs: - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} shell: bash -el {0} working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE - env: - CONDA_SOLVER: libmamba # CONDA-LIBMAMBA-SOLVER CHANGE run: | ./dev/macos/${{ matrix.test-type }}.sh - uses: codecov/codecov-action@v3 with: - flags: ${{ matrix.test-type }} - env_vars: OS,PYTHON + flags: ${{ matrix.test-type }},${{ runner.os }},${{ matrix.python-version }} - name: Upload test results - if: always() + if: '!cancelled()' uses: actions/upload-artifact@v3 with: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths path: | conda/.coverage - conda/.test_durations_${OS} + conda/tools/durations/${{ runner.os }}.json conda/test-report.xml retention-days: 1 # aggregate and upload aggregate: # only aggregate test suite if there are code changes - needs: [changes, windows, linux, linux-qemu, macos] - if: needs.changes.outputs.code == 'true' || github.event_name == 'schedule' + needs: [changes, windows, linux, macos] + if: (!cancelled() && needs.changes.outputs.code == 'true') || github.event_name == 'schedule' runs-on: ubuntu-latest steps: @@ -505,8 +397,8 @@ jobs: # required check analyze: name: Analyze results - needs: [windows, linux, linux-qemu, macos, aggregate] - if: always() + needs: [windows, linux, macos, aggregate] + if: '!cancelled()' runs-on: ubuntu-latest steps: @@ -533,51 +425,3 @@ jobs: filename: .github/TEST_FAILURE_REPORT_TEMPLATE.md update_existing: true # /CONDA-LIBMAMBA-SOLVER CHANGE - - # canary builds - build: - name: Canary Build - needs: [analyze] - # only build canary build if - # - prior steps succeeded, - # - this is the main repo, and - # - event triggered by push, and # CONDA-LIBMAMBA-SOLVER CHANGE - # - we are on the main (or feature) branch - if: >- - success() - && !github.event.repository.fork - && github.event_name == 'push' - && ( - github.ref_name == 'main' - || startsWith(github.ref_name, 'feature/') - ) - strategy: - matrix: - include: - - runner: ubuntu-latest - subdir: linux-64 - - runner: macos-latest - subdir: osx-64 - - runner: windows-latest - subdir: win-64 - runs-on: ${{ matrix.runner }} - steps: - # Clean checkout of specific git ref needed for package metadata version - # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR: - - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - clean: true - fetch-depth: 0 - - - name: Create and upload canary build - uses: conda/actions/canary-release@v22.10.0 - env: - # Run conda-build in isolated activation to properly package conda - _CONDA_BUILD_ISOLATED_ACTIVATION: 1 - with: - package-name: ${{ github.event.repository.name }} - subdir: ${{ matrix.subdir }} - anaconda-org-channel: conda-canary - anaconda-org-label: ${{ github.ref_name == 'main' && 'dev' || github.ref_name }} - anaconda-org-token: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2885de60..ff812c4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,6 @@ ci: exclude: | (?x)^( tests/data/ | - tests/_reposerver\.py | conda_libmamba_solver/mamba_utils\.py )/ repos: @@ -29,7 +28,7 @@ repos: args: ["--py38-plus"] exclude: ^conda/exports.py - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black exclude: tests/_reposerver\.py @@ -65,4 +64,4 @@ repos: - id: insert-license files: \.py$ args: [--license-filepath, .github/disclaimer.txt, --no-extra-eol] - exclude: ^(tests/repodata_time_machine.py|mamba_utils\.py) # extend global exclude + exclude: ^(tests/repodata_time_machine.py|mamba_utils\.py|tests/channel_testing/helpers\.py|tests/channel_testing/reposerver\.py) # extend global exclude diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index c42d0ba6..f5e64e78 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -156,12 +156,14 @@ def reload_local_channels(self): """ Reload a channel that was previously loaded from a local directory. """ - for url, info in self._index.items(): - if url.startswith("file://"): - url, json_path, overlay_path = self._fetch_channel(url) - new = self._json_path_to_repo_info(url, json_path, overlay_path) - self._repos[self._repos.index(info.repo)] = new.repo - self._index[url] = new + for noauth_url, info in self._index.items(): + if noauth_url.startswith("file://") or info.channel.scheme == "file": + url, json_path = self._fetch_channel(info.full_url) + repo_position = self._repos.index(info.repo) + info.repo.clear(True) + new = self._json_path_to_repo_info(url, json_path, try_solv=False) + self._repos[repo_position] = new.repo + self._index[noauth_url] = new set_channel_priorities(self._index) def _repo_from_records( @@ -245,21 +247,25 @@ def _fetch_channel(self, url: str) -> tuple[str, Path, Path | None]: return url, json_path, overlay_path def _json_path_to_repo_info( - self, url: str, json_path: str | Path, overlay_path: Path | None = None - ) -> _ChannelRepoInfo | None: + self, url: str, json_path: str, try_solv: bool = False + ) -> Optional[_ChannelRepoInfo]: channel = Channel.from_url(url) noauth_url = channel.urls(with_credentials=False, subdirs=(channel.subdir,))[0] json_path = Path(json_path) - solv_path = json_path.parent / f"{json_path.stem}.solv" try: json_stat = json_path.stat() except OSError as exc: log.debug("Failed to stat %s", json_path, exc_info=exc) json_stat = None - try: - solv_stat = solv_path.stat() - except OSError as exc: - log.debug("Failed to stat %s", solv_path, exc_info=exc) + if try_solv: + try: + solv_path = json_path.parent / f"{json_path.stem}.solv" + solv_stat = solv_path.stat() + except OSError as exc: + log.debug("Failed to stat %s", solv_path, exc_info=exc) + solv_stat = None + else: + solv_path = None solv_stat = None if solv_stat is None and json_stat is None: @@ -304,8 +310,9 @@ def _load_channels(self) -> dict[str, _ChannelRepoInfo]: noauth_urls = c.urls(with_credentials=False, subdirs=self._subdirs) if seen_noauth.issuperset(noauth_urls): continue - if c.auth or c.token: # authed channel always takes precedence - urls += Channel(c).urls(with_credentials=True, subdirs=self._subdirs) + auth_urls = c.urls(with_credentials=True, subdirs=self._subdirs) + if noauth_urls != auth_urls: # authed channel always takes precedence + urls += auth_urls seen_noauth.update(noauth_urls) continue # at this point, we are handling an unauthed channel; in some edge cases, diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 6b14203f..d52bc713 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -26,9 +26,9 @@ REPODATA_FN, UNKNOWN_CHANNEL, ChannelPriority, - on_win, ) from conda.base.context import context +from conda.common.compat import on_win from conda.common.constants import NULL from conda.common.io import Spinner, timeout from conda.common.path import paths_equal @@ -160,7 +160,6 @@ def solve_final_state( # From now on we _do_ require a solver and the index init_api_context() subdirs = self.subdirs - conda_bld_channels = () if self._called_from_conda_build(): log.info("Using solver via 'conda.plan.install_actions' (probably conda build)") # Problem: Conda build generates a custom index which happens to "forget" about @@ -179,6 +178,7 @@ def solve_final_state( IndexHelper = _CachedLibMambaIndexHelper else: IndexHelper = LibMambaIndexHelper + conda_bld_channels = () all_channels = [ *conda_bld_channels, @@ -402,10 +402,16 @@ def _solve_attempt( def _specs_to_tasks(self, in_state: SolverInputState, out_state: SolverOutputState): log.debug("Creating tasks for %s specs", len(out_state.specs)) if in_state.is_removing: - return self._specs_to_tasks_remove(in_state, out_state) - if self._called_from_conda_build(): - return self._specs_to_tasks_conda_build(in_state, out_state) - return self._specs_to_tasks_add(in_state, out_state) + tasks = self._specs_to_tasks_remove(in_state, out_state) + elif self._called_from_conda_build(): + tasks = self._specs_to_tasks_conda_build(in_state, out_state) + else: + tasks = self._specs_to_tasks_add(in_state, out_state) + log.debug( + "Created following tasks:\n%s", + json.dumps({k[0]: v for k, v in tasks.items()}, indent=2), + ) + return tasks @staticmethod def _spec_to_str(spec): @@ -456,7 +462,7 @@ def _specs_to_tasks_add(self, in_state: SolverInputState, out_state: SolverOutpu # logic considers should be the target version for each package in the environment # and requested changes. We are _not_ following those targets here, but we do iterate # over the list to decide what to do with each package. - for name, _classic_logic_spec in out_state.specs.items(): + for name, _classic_logic_spec in sorted(out_state.specs.items()): if name.startswith("__"): continue # ignore virtual packages installed: PackageRecord = in_state.installed.get(name) @@ -820,8 +826,11 @@ def _export_solved_records( else: log.warn("Tried to unlink %s but it is not installed or manageable?", filename) + for_conda_build = self._called_from_conda_build() for channel, filename, json_payload in to_link: - record = self._package_record_from_json_payload(index, channel, filename, json_payload) + record = self._package_record_from_json_payload( + index, channel, filename, json_payload, for_conda_build=for_conda_build + ) # We need this check below to make sure noarch package get reinstalled # record metadata coming from libmamba is incomplete and won't pass the # noarch checks -- to fix it, we swap the metadata-only record with its locally @@ -842,20 +851,28 @@ def _export_solved_records( ) # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces - if on_win and self._called_from_conda_build(): + if on_win and for_conda_build: for record in out_state.records.values(): - record.channel.location = percent_decode(record.channel.location) + if "%" not in str(record): + continue + if record.channel.location: # multichannels like 'defaults' have no location + record.channel.location = percent_decode(record.channel.location) record.channel.name = percent_decode(record.channel.name) def _package_record_from_json_payload( - self, index: LibMambaIndexHelper, channel: str, pkg_filename: str, json_payload: str + self, + index: LibMambaIndexHelper, + channel: str, + pkg_filename: str, + json_payload: str, + for_conda_build: bool = False, ) -> PackageRecord: """ The libmamba transactions cannot return full-blown objects from the C/C++ side. Instead, it returns the instructions to build one on the Python side: channel_info: dict - Channel data, as built in .index.LibmambaIndexHelper._fetch_channel() + Channel datas, as built in .index.LibmambaIndexHelper._fetch_channel() This is retrieved from the .index._index mapping, keyed by channel URLs pkg_filename: str The filename (.tar.bz2 or .conda) of the selected record. @@ -881,6 +898,14 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename kwargs["channel"] = channel_info.channel + if for_conda_build: + # conda-build expects multichannel instances in the Dist->PackageRecord mapping + # see https://github.com/conda/conda-libmamba-solver/issues/363 + for multichannel_name, mc_channels in context.custom_multichannels.items(): + urls = [url for c in mc_channels for url in c.urls(with_credentials=False)] + if channel_info.noauth_url in urls: + kwargs["channel"] = multichannel_name + break kwargs["url"] = join_url(channel_info.full_url, pkg_filename) if not kwargs.get("subdir"): # missing in old channels kwargs["subdir"] = channel_info.channel.subdir diff --git a/conda_libmamba_solver/state.py b/conda_libmamba_solver/state.py index dd6edc47..93acb2c6 100644 --- a/conda_libmamba_solver/state.py +++ b/conda_libmamba_solver/state.py @@ -231,8 +231,9 @@ def installed(self) -> Mapping[str, PackageRecord]: """ This exposes the installed packages in the prefix. Note that a ``PackageRecord`` can generate an equivalent ``MatchSpec`` object with ``.to_match_spec()``. + Records are toposorted. """ - return MappingProxyType(self.prefix_data._prefix_records) + return MappingProxyType(dict(sorted(self.prefix_data._prefix_records.items()))) @property def history(self) -> Mapping[str, MatchSpec]: @@ -261,7 +262,7 @@ def virtual(self) -> Mapping[str, MatchSpec]: cannot be (un)installed, they only represent constrains for other packages. By convention, their names start with a double underscore. """ - return MappingProxyType(self._virtual) + return MappingProxyType(dict(sorted(self._virtual.items()))) @property def aggressive_updates(self) -> Mapping[str, MatchSpec]: @@ -281,12 +282,14 @@ def always_update(self) -> Mapping[str, MatchSpec]: - almost all packages if update_all is true - etc """ - pkgs = {pkg: MatchSpec(pkg) for pkg in self.aggressive_updates if pkg in self.installed} + installed = self.installed + pinned = self.pinned + pkgs = {pkg: MatchSpec(pkg) for pkg in self.aggressive_updates if pkg in installed} if context.auto_update_conda and paths_equal(self.prefix, context.root_prefix): pkgs.setdefault("conda", MatchSpec("conda")) if self.update_modifier.UPDATE_ALL: - for pkg in self.installed: - if pkg != "python" and pkg not in self.pinned: + for pkg in installed: + if pkg != "python" and pkg not in pinned: pkgs.setdefault(pkg, MatchSpec(pkg)) return MappingProxyType(pkgs) diff --git a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py deleted file mode 100644 index f10fbf86..00000000 --- a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (C) 2022 Anaconda, Inc -# Copyright (C) 2023 conda -# SPDX-License-Identifier: BSD-3-Clause -""" -pytest plugin to modify which upstream (conda/conda) tests are run by pytest. -""" - -# Deselect tests from conda/conda we cannot pass due to different reasons -# These used to be skipped or xfail'd upstream, but we are trying to -# keep it clean from this project's specifics -_deselected_upstream_tests = { - # This test checks for plugin errors and assumes none are present, but - # conda-libmamba-solver counts as one so we need to skip it. - "tests/plugins/test_manager.py": ["test_load_entrypoints_importerror"], - # Conflict report / analysis is done differently with libmamba. - "tests/cli/test_cli_install.py": ["test_find_conflicts_called_once"], - "tests/core/test_solve.py": [ - # SolverStateContainer needed - "test_solve_2", - "test_virtual_package_solver", - "test_broken_install", - # Features / nomkl involved - "test_features_solve_1", - "test_prune_1", - "test_update_prune_2", - "test_update_prune_3", - # Message expected, but libmamba does not report constraints - "test_update_prune_5", - # classic expects implicit update to channel with higher priority, including downgrades - # libmamba does not do this, it just stays in the same channel; should it change? - "test_priority_1", - # FIXME: Known issue: We can use a VERIFY task, but that causes a "dance" across solves, - # where the verification task changes a few specs. Next time it runs it undoes it. - "test_force_remove_1", - # The following are known to fail upstream due to too strict expectations - # We provide the same tests with adjusted checks in tests/test_modified_upstream.py - "test_pinned_1", - "test_freeze_deps_1", - "test_cuda_fail_1", - "test_cuda_fail_2", - "test_update_all_1", - "test_conda_downgrade", - "test_python2_update", - "test_fast_update_with_update_modifier_not_set", - "test_downgrade_python_prevented_with_sane_message", - ], - "tests/test_create.py": [ - # libmamba does not support features - "test_remove_features", - # Known bug in mamba; see https://github.com/mamba-org/mamba/issues/1197 - "test_offline_with_empty_index_cache", - # Adjusted in tests/test_modified_upstream.py - "test_install_features", - # libmamba departs from this behavior in the classic logic - # see https://github.com/conda/conda-libmamba-solver/pull/289 - "test_pinned_override_with_explicit_spec", - # TODO: https://github.com/conda/conda-libmamba-solver/issues/141 - "test_conda_pip_interop_conda_editable_package", - ], - # These use libmamba-incompatible MatchSpecs (name[build_number=1] syntax) - "tests/models/test_prefix_graph.py": [ - "test_deep_cyclical_dependency", - # TODO: Investigate this, since they are solver related-ish - "test_windows_sort_orders_1", - ], - # See https://github.com/conda/conda-libmamba-solver/pull/133#issuecomment-1448607110 - # These failed after enabling the whole unit test suite for `conda/conda`. - # Errors are not critical but would require some further assessment in case fixes are obvious. - "tests/cli/test_main_notices.py": [ - "test_notices_appear_once_when_running_decorated_commands", - "test_notices_does_not_interrupt_command_on_failure", - ], - "tests/conda_env/installers/test_pip.py": [ - "PipInstallerTest::test_stops_on_exception", - "PipInstallerTest::test_straight_install", - ], - "tests/conda_env/specs/test_base.py": [ - "DetectTestCase::test_build_msg", - "DetectTestCase::test_dispatches_to_registered_specs", - "DetectTestCase::test_has_build_msg_function", - "DetectTestCase::test_passes_kwargs_to_all_specs", - "DetectTestCase::test_raises_exception_if_no_detection", - ], - # TODO: Known issue: https://github.com/conda/conda-libmamba-solver/issues/320 - "tests/conda_env/test_cli.py": [ - "test_update_env_no_action_json_output", - "test_update_env_only_pip_json_output", - ], - # TODO: Fix upstream; they seem to assume no other solvers will be active via env var - "tests/plugins/test_solvers.py": [ - "test_get_solver_backend", - "test_get_solver_backend_multiple", - ], - # TODO: Investigate these, since they are solver related-ish - "tests/conda_env/specs/test_requirements.py": [ - "TestRequirements::test_environment", - ], - # Added to test_modified_upstream.py - "tests/test_priority.py": ["test_reorder_channel_priority"], - # Added to test_modified_upstream.py; this passes just by moving it to another test file - "tests/test_misc.py": ["test_explicit_missing_cache_entries"], - # Unrelated to libmamba, but we need to skip it because it fails in CI - "tests/test_activate.py": ["test_bash_basic_integration"], -} - - -def pytest_collection_modifyitems(session, config, items): - """ - We use this hook to modify which upstream tests (from the conda/conda repo) - are run by pytest. - - This hook should not return anything but, instead, modify in place. - """ - selected = [] - deselected = [] - for item in items: - path_key = "/".join(item.path.parts[item.path.parts.index("tests") :]) - item_name_no_brackets = item.name.split("[")[0] - if item_name_no_brackets in _deselected_upstream_tests.get(path_key, []): - deselected.append(item) - continue - selected.append(item) - items[:] = selected - config.hook.pytest_deselected(items=deselected) diff --git a/dev/collect_upstream_conda_tests/pyproject.toml b/dev/collect_upstream_conda_tests/pyproject.toml deleted file mode 100644 index b650d998..00000000 --- a/dev/collect_upstream_conda_tests/pyproject.toml +++ /dev/null @@ -1,33 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "collect-upstream-conda-tests" -version = "0.0.1" -description = "A pytest plugin to filter which upstream tests are run" -authors = [ - {name = "Anaconda, Inc.", email = "conda@continuum.io"} -] -license = {file = "../../LICENSE"} -classifiers = [ - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy" -] -requires-python = ">=3.8" -dependencies = [ - "pytest", -] - -[project.urls] -homepage = "https://github.com/conda/conda-libmamba-solver" - -[project.entry-points.pytest11] -collect-upstream-conda-tests = "collect_upstream_conda_tests" diff --git a/dev/linux/bashrc.sh b/dev/linux/bashrc.sh index fdceb01a..cb04d122 100644 --- a/dev/linux/bashrc.sh +++ b/dev/linux/bashrc.sh @@ -44,7 +44,6 @@ if [ -d "/opt/mamba-src" ]; then fi cd /opt/conda-libmamba-solver-src -sudo /opt/conda/bin/python -m pip install ./dev/collect_upstream_conda_tests/ --no-deps sudo /opt/conda/bin/python -m pip install -e . --no-deps cd /opt/conda-src diff --git a/dev/linux/upstream_integration.sh b/dev/linux/upstream_integration.sh index d614071f..d875629c 100755 --- a/dev/linux/upstream_integration.sh +++ b/dev/linux/upstream_integration.sh @@ -20,7 +20,6 @@ sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repo --file "${CONDA_SRC}/tests/requirements.txt" \ --file "${CONDA_SRC}/tests/requirements-s3.txt" \ --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt" -sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/" sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC" --no-deps -vvv # /CONDA LIBMAMBA SOLVER CHANGES eval "$(sudo /opt/conda/bin/python -m conda init --dev bash)" diff --git a/dev/linux/upstream_unit.sh b/dev/linux/upstream_unit.sh index 649a24af..0112ebed 100755 --- a/dev/linux/upstream_unit.sh +++ b/dev/linux/upstream_unit.sh @@ -20,7 +20,6 @@ sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repo --file "${CONDA_SRC}/tests/requirements.txt" \ --file "${CONDA_SRC}/tests/requirements-s3.txt" \ --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt" -sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/" sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC" --no-deps -vvv # /CONDA LIBMAMBA SOLVER CHANGES diff --git a/docs/dev/setup.md b/docs/dev/setup.md index e6c88866..20d3a7aa 100644 --- a/docs/dev/setup.md +++ b/docs/dev/setup.md @@ -77,13 +77,6 @@ $ cd $REPO_LOCATION $ python -m pip install --no-deps -e . ``` -5. Install the test collection plugins (only for upstream tests in `conda/conda`): - -```bash -$ cd $REPO_LOCATION -$ python -m pip install dev/collect_upstream_conda_tests/ -``` - For testing out the `libmamba` solve you can set it several ways: - environment variable `CONDA_SOLVER=libmamba` - pass a flag `--solver=libmamba` diff --git a/docs/dev/workflows.md b/docs/dev/workflows.md index 1bbe9e22..49d52640 100644 --- a/docs/dev/workflows.md +++ b/docs/dev/workflows.md @@ -27,6 +27,3 @@ From the properly mounted `conda/conda` Docker container (see ["Development envi $ cd /opt/conda-src $ CONDA_SOLVER=libmamba pytest ``` - -Note we [deselect some upstream tests in our `pyproject.toml`](../../dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py) for a number of reasons. -For this to work we need to ensure that `pytest` loads that plugin by installing it in the same environment. diff --git a/news/365-canonical-channel-names b/news/365-canonical-channel-names new file mode 100644 index 00000000..58a9b4a1 --- /dev/null +++ b/news/365-canonical-channel-names @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Use canonical channel names (if available) in exported `PackageRecord` objects. Fixes an issue with conda-build and custom multichannels. (#363 via #365) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/news/366-auth-multichannel b/news/366-auth-multichannel new file mode 100644 index 00000000..ed356e44 --- /dev/null +++ b/news/366-auth-multichannel @@ -0,0 +1,20 @@ +### Enhancements + +* + +### Bug fixes + +* Allow authenticated URLs in `default_channels` and other multichannels. (#364 via #366) +* Preserve authentication while reloading local channels. (#366) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/news/378-sort-installed b/news/378-sort-installed new file mode 100644 index 00000000..5fc31e4b --- /dev/null +++ b/news/378-sort-installed @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Ensure specs, installed and virtual packages are always sorted to avoid injecting accidental randomness in the solve process. (#378) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/channel_testing/helpers.py b/tests/channel_testing/helpers.py new file mode 100644 index 00000000..549e05f7 --- /dev/null +++ b/tests/channel_testing/helpers.py @@ -0,0 +1,169 @@ +# Copyright (C) 2019 QuantStack and the Mamba contributors. +# Copyright (C) 2022 Anaconda, Inc +# Copyright (C) 2023 conda +# SPDX-License-Identifier: BSD-3-Clause +import os +import pathlib +import socket +import subprocess +import sys +from typing import Tuple + +import pytest +from conda.testing.integration import _get_temp_prefix, run_command +from xprocess import ProcessStarter + + +def _dummy_http_server(xprocess, name, port, auth="none", user=None, password=None, token=None): + """ + Adapted from + https://github.com/mamba-org/powerloader/blob/effe2b7e1/test/helpers.py#L11 + """ + curdir = pathlib.Path(__file__).parent + print("Starting dummy_http_server") + + class Starter(ProcessStarter): + pattern = f"Server started at localhost:{port}" + terminate_on_interrupt = True + timeout = 10 + args = [ + sys.executable, + "-u", # unbuffered + str(curdir / "reposerver.py"), + "-d", + str(curdir / ".." / "data" / "mamba_repo"), + "--port", + str(port), + ] + if auth == "token": + assert token + args += ["--token", token] + elif auth: + args += ["--auth", auth] + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + if user and password: + env["TESTPWD"] = f"{user}:{password}" + + def startup_check(self): + s = socket.socket() + address = "localhost" + error = False + try: + s.connect((address, port)) + except Exception as e: + print("something's wrong with %s:%d. Exception is %s" % (address, port, e)) + error = True + finally: + s.close() + + return not error + + logfile = xprocess.ensure(name, Starter) + + if user and password: + yield f"http://{user}:{password}@localhost:{port}" + elif token: + yield f"http://localhost:{port}/t/{token}" + else: + yield f"http://localhost:{port}" + + xprocess.getinfo(name).terminate() + + +@pytest.fixture +def http_server_auth_none(xprocess): + yield from _dummy_http_server(xprocess, name="http_server_auth_none", port=8000, auth="none") + + +@pytest.fixture +def http_server_auth_none_debug_repodata(xprocess): + yield from _dummy_http_server( + xprocess, + name="http_server_auth_none_debug_repodata", + port=8000, + auth="none-debug-repodata", + ) + + +@pytest.fixture +def http_server_auth_none_debug_packages(xprocess): + yield from _dummy_http_server( + xprocess, + name="http_server_auth_none_debug_packages", + port=8000, + auth="none-debug-packages", + ) + + +@pytest.fixture +def http_server_auth_basic(xprocess): + yield from _dummy_http_server( + xprocess, + name="http_server_auth_basic", + port=8000, + auth="basic", + user="user", + password="test", + ) + + +@pytest.fixture +def http_server_auth_basic_email(xprocess): + yield from _dummy_http_server( + xprocess, + name="http_server_auth_basic_email", + port=8000, + auth="basic", + user="user@email.com", + password="test", + ) + + +@pytest.fixture +def http_server_auth_token(xprocess): + yield from _dummy_http_server( + xprocess, + name="http_server_auth_token", + port=8000, + auth="token", + token="xy-12345678-1234-1234-1234-123456789012", + ) + + +def create_with_channel( + channel, solver="libmamba", check=True, **kwargs +) -> subprocess.CompletedProcess: + return subprocess.run( + [ + sys.executable, + "-m", + "conda", + "create", + "-p", + _get_temp_prefix(), + f"--solver={solver}", + "--json", + "--override-channels", + "-c", + channel, + "test-package", + ], + check=check, + **kwargs, + ) + + +def create_with_channel_in_process(channel, solver="libmamba", **kwargs) -> Tuple[str, str, int]: + stdout, stderr, returncode = run_command( + "create", + _get_temp_prefix(), + f"--solver={solver}", + "--json", + "--override-channels", + "-c", + channel, + "test-package", + **kwargs, + ) + return stdout, stderr, returncode diff --git a/tests/channel_testing/reposerver.py b/tests/channel_testing/reposerver.py new file mode 100644 index 00000000..b058d47e --- /dev/null +++ b/tests/channel_testing/reposerver.py @@ -0,0 +1,355 @@ +# Copyright (C) 2019 QuantStack and the Mamba contributors. +# Copyright (C) 2022 Anaconda, Inc +# Copyright (C) 2023 conda +# SPDX-License-Identifier: BSD-3-Clause +""" +Helper module/script to launch a conda channel/server +for local testing. + +Copied from https://github.com/mamba-org/mamba/blob/53eb28d/mamba/tests/reposerver.py +on Apr 27 2022 + +See data/mamba_repo/LICENSE for full details +""" +import argparse +import base64 +import glob +import os +import re +import shutil +from http.server import HTTPServer, SimpleHTTPRequestHandler +from pathlib import Path + +try: + import conda_content_trust.authentication as cct_authentication + import conda_content_trust.common as cct_common + import conda_content_trust.metadata_construction as cct_metadata_construction + import conda_content_trust.root_signing as cct_root_signing + import conda_content_trust.signing as cct_signing + + conda_content_trust_available = True +except ImportError: + conda_content_trust_available = False + +if os.environ.get("TESTPWD"): + default_user, default_password = os.environ.get("TESTPWD").split(":") +else: + default_user, default_password = None, None + +parser = argparse.ArgumentParser(description="Start a simple conda package server.") +parser.add_argument("-p", "--port", type=int, default=8000, help="Port to use.") +parser.add_argument( + "-d", + "--directory", + type=str, + default=os.getcwd(), + help="Root directory for serving.", +) +parser.add_argument( + "-a", + "--auth", + default=None, + type=str, + help="auth method (none, none-debug-repodata, none-debug-packages, basic, or token)", +) +parser.add_argument( + "--sign", + action="store_true", + help="Sign repodata (note: run generate_gpg_keys.sh before)", +) +parser.add_argument( + "--token", + type=str, + default=None, + help="Use token as API Key", +) +parser.add_argument( + "--user", + type=str, + default=default_user, + help="Use token as API Key", +) +parser.add_argument( + "--password", + type=str, + default=default_password, + help="Use token as API Key", +) +args = parser.parse_args() + + +def get_fingerprint(gpg_output): + lines = gpg_output.splitlines() + fpline = lines[1].strip() + fpline = fpline.replace(" ", "") + return fpline + + +class RepoSigner: + keys = { + "root": [], + "key_mgr": [ + { + "private": "c9c2060d7e0d93616c2654840b4983d00221d8b6b69c850107da74b42168f937", + "public": "013ddd714962866d12ba5bae273f14d48c89cf0773dee2dbf6d4561e521c83f7", + }, + ], + "pkg_mgr": [ + { + "private": "f3cdab14740066fb277651ec4f96b9f6c3e3eb3f812269797b9656074cd52133", + "public": "f46b5a7caa43640744186564c098955147daa8bac4443887bc64d8bfee3d3569", + } + ], + } + + def normalize_keys(self, keys): + out = {} + for ik, iv in keys.items(): + out[ik] = [] + for el in iv: + if isinstance(el, str): + el = el.lower() + keyval = cct_root_signing.fetch_keyval_from_gpg(el) + res = {"fingerprint": el, "public": keyval} + elif isinstance(el, dict): + res = { + "private": el["private"].lower(), + "public": el["public"].lower(), + } + out[ik].append(res) + + return out + + def create_root(self, keys): + root_keys = keys["root"] + + root_pubkeys = [k["public"] for k in root_keys] + key_mgr_pubkeys = [k["public"] for k in keys["key_mgr"]] + + root_version = 1 + + root_md = cct_metadata_construction.build_root_metadata( + root_pubkeys=root_pubkeys[0:1], + root_threshold=1, + root_version=root_version, + key_mgr_pubkeys=key_mgr_pubkeys, + key_mgr_threshold=1, + ) + + # Wrap the metadata in a signing envelope. + root_md = cct_signing.wrap_as_signable(root_md) + + root_md_serialized_unsigned = cct_common.canonserialize(root_md) + + root_filepath = self.folder / f"{root_version}.root.json" + print("Writing out: ", root_filepath) + # Write unsigned sample root metadata. + with open(root_filepath, "wb") as fout: + fout.write(root_md_serialized_unsigned) + + # This overwrites the file with a signed version of the file. + cct_root_signing.sign_root_metadata_via_gpg(root_filepath, root_keys[0]["fingerprint"]) + + # Load untrusted signed root metadata. + signed_root_md = cct_common.load_metadata_from_file(root_filepath) + + cct_authentication.verify_signable(signed_root_md, root_pubkeys, 1, gpg=True) + + print("[reposigner] Root metadata signed & verified!") + + def create_key_mgr(self, keys): + private_key_key_mgr = cct_common.PrivateKey.from_hex(keys["key_mgr"][0]["private"]) + pkg_mgr_pub_keys = [k["public"] for k in keys["pkg_mgr"]] + key_mgr = cct_metadata_construction.build_delegating_metadata( + metadata_type="key_mgr", # 'root' or 'key_mgr' + delegations={"pkg_mgr": {"pubkeys": pkg_mgr_pub_keys, "threshold": 1}}, + version=1, + # timestamp default: now + # expiration default: now plus root expiration default duration + ) + + key_mgr = cct_signing.wrap_as_signable(key_mgr) + + # sign dictionary in place + cct_signing.sign_signable(key_mgr, private_key_key_mgr) + + key_mgr_serialized = cct_common.canonserialize(key_mgr) + with open(self.folder / "key_mgr.json", "wb") as fobj: + fobj.write(key_mgr_serialized) + + # let's run a verification + root_metadata = cct_common.load_metadata_from_file(self.folder / "1.root.json") + key_mgr_metadata = cct_common.load_metadata_from_file(self.folder / "key_mgr.json") + + cct_common.checkformat_signable(root_metadata) + + if "delegations" not in root_metadata["signed"]: + raise ValueError('Expected "delegations" entry in root metadata.') + + root_delegations = root_metadata["signed"]["delegations"] # for brevity + cct_common.checkformat_delegations(root_delegations) + if "key_mgr" not in root_delegations: + raise ValueError('Missing expected delegation to "key_mgr" in root metadata.') + cct_common.checkformat_delegation(root_delegations["key_mgr"]) + + # Doing delegation processing. + cct_authentication.verify_delegation("key_mgr", key_mgr_metadata, root_metadata) + + print("[reposigner] success: key mgr metadata verified based on root metadata.") + + return key_mgr + + def sign_repodata(self, repodata_fn, keys): + target_folder = self.folder / repodata_fn.parent.name + if not target_folder.exists(): + target_folder.mkdir() + + final_fn = target_folder / repodata_fn.name + print("copy", repodata_fn, final_fn) + shutil.copyfile(repodata_fn, final_fn) + + pkg_mgr_key = keys["pkg_mgr"][0]["private"] + cct_signing.sign_all_in_repodata(str(final_fn), pkg_mgr_key) + print(f"[reposigner] Signed {final_fn}") + + def __init__(self, in_folder=args.directory): + self.keys["root"] = [ + get_fingerprint(os.environ["KEY1"]), + get_fingerprint(os.environ["KEY2"]), + ] + + self.in_folder = Path(in_folder).resolve() + self.folder = self.in_folder.parent / (str(self.in_folder.name) + "_signed") + + if not self.folder.exists(): + os.mkdir(self.folder) + + self.keys = self.normalize_keys(self.keys) + print("[reposigner] Using keys:", self.keys) + + print("[reposigner] Using folder:", self.folder) + + self.create_root(self.keys) + self.create_key_mgr(self.keys) + for f in glob.glob(str(self.in_folder / "**" / "repodata.json")): + self.sign_repodata(Path(f), self.keys) + + +class RepodataHeadersHandler(SimpleHTTPRequestHandler): + """ + This handler is used to debug requests to repodata.json files. We + make them error out with a failed redirection to a URL that + contains the client headers in the URL, encoded as base64. + """ + + path_suffix_to_debug = "repodata.json" + + def do_GET(self) -> None: + """ + HACK: if a repodata.json is requested, redirect + to a fake address which encodes the client headers + as b64. This way, we can parse the exception message in a test. + """ + if not self.path.endswith(self.path_suffix_to_debug): + return super().do_GET() + headers_b64 = base64.b64encode(str(self.headers).encode("utf-8")) + self.send_response(307) # redirect + self.send_header("Location", f"/headers/{headers_b64.decode('utf-8')}") + self.end_headers() + + +class PackagesHeadersHandler(RepodataHeadersHandler): + "Same as RepodataHeadersHandler, but it fails when tarballs are requested" + path_suffix_to_debug = ".tar.bz2" + + +class BasicAuthHandler(SimpleHTTPRequestHandler): + """Main class to present webpages and authentication.""" + + user = args.user + password = args.password + key = base64.b64encode(bytes(f"{args.user}:{args.password}", "utf-8")).decode("ascii") + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_AUTHHEAD(self): + self.send_response(401) + self.send_header("WWW-Authenticate", 'Basic realm="Test"') + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_GET(self): + """Present frontpage with user authentication.""" + auth_header = self.headers.get("Authorization", "") + + if not auth_header: + self.do_AUTHHEAD() + self.wfile.write(b"no auth header received") + pass + elif auth_header == "Basic " + self.key: + SimpleHTTPRequestHandler.do_GET(self) + pass + else: + self.do_AUTHHEAD() + self.wfile.write(auth_header.encode("ascii")) + self.wfile.write(b"not authenticated") + pass + + +class CondaTokenHandler(SimpleHTTPRequestHandler): + """Main class to present webpages and authentication.""" + + api_key = args.token + token_pattern = re.compile("^/t/([^/]+?)/") + + def do_GET(self): + """Present frontpage with user authentication.""" + match = self.token_pattern.search(self.path) + if match: + prefix_length = len(match.group(0)) - 1 + new_path = self.path[prefix_length:] + found_api_key = match.group(1) + if found_api_key == self.api_key: + self.path = new_path + return SimpleHTTPRequestHandler.do_GET(self) + + self.send_response(403) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(b"no valid api key received") + + +if args.sign: + if not conda_content_trust_available: + print("Conda content trust not installed!") + exit(1) + signer = RepoSigner() + os.chdir(signer.folder) +else: + os.chdir(args.directory) + +if args.auth == "none": + handler = SimpleHTTPRequestHandler +elif args.auth == "none-debug-repodata": + handler = RepodataHeadersHandler +elif args.auth == "none-debug-packages": + handler = PackagesHeadersHandler +elif args.auth == "basic" or (args.user and args.password): + handler = BasicAuthHandler +elif args.auth == "token" or args.token: + handler = CondaTokenHandler + +PORT = args.port + +server = HTTPServer(("", PORT), handler) +print("Server started at localhost:" + str(PORT)) +try: + server.serve_forever() +except Exception as exc: + # Catch all sorts of interrupts + print("Shutting server down:", exc.__class__.__name__, exc) + server.shutdown() + print("Server shut down") diff --git a/tests/data/conda_build_recipes/stackvana/meta.yaml b/tests/data/conda_build_recipes/stackvana/meta.yaml index 24286527..2991ce80 100644 --- a/tests/data/conda_build_recipes/stackvana/meta.yaml +++ b/tests/data/conda_build_recipes/stackvana/meta.yaml @@ -1,5 +1,6 @@ {% set name = "stackvana-core" %} {% set version = "0.2021.43" %} +{% set eups_product = "lsst_distrib" %} package: name: {{ name|lower }} @@ -15,11 +16,45 @@ outputs: script: - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl # [unix] - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl # [win] + test: + commands: + - echo OK - name: stackvana-core version: {{ version }} - run_exports: - - {{ pin_subpackage('stackvana-core-impl', exact=True) }} - + build: + script: + - echo "BUILDING CORE" >> $PREFIX/stackvana-core # [unix] + - echo "BUILDING CORE" >> %PREFIX%/stackvana-core # [win] + run_exports: + - {{ pin_subpackage('stackvana-core-impl', exact=True) }} requirements: run: - {{ pin_subpackage('stackvana-core-impl', exact=True) }} + test: + commands: + - echo OK + - name: stackvana-{{ eups_product }} + version: {{ version }} + build: + script: + - echo "BUILDING {{ eups_product }}" >> $PREFIX/stackvana-{{ eups_product }} # [unix] + - echo "BUILDING {{ eups_product }}" >> %PREFIX%/stackvana-{{ eups_product }} # [win] + requirements: + host: + - stackvana-core =={{ version }} + run: + - stackvana-core =={{ version }} + test: + commands: + - echo OK + - name: stackvana + version: {{ version }} + build: + script: + - echo "BUILDING STACKVANA" >> $PREFIX/stackvana # [unix] + - echo "BUILDING STACKVANA" >> %PREFIX%/stackvana # [win] + requirements: + - {{ pin_subpackage("stackvana-" ~ eups_product, max_pin="x.x.x") }} + test: + commands: + - echo OK diff --git a/tests/test_channels.py b/tests/test_channels.py index c3f6bae4..ed296608 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -5,19 +5,26 @@ import os import shutil import sys -from datetime import datetime from pathlib import Path import pytest -from conda.common.compat import on_linux +from conda.base.context import reset_context +from conda.common.compat import on_linux, on_win from conda.common.io import env_vars from conda.core.prefix_data import PrefixData from conda.models.channel import Channel from conda.testing.integration import _get_temp_prefix, make_temp_env from conda.testing.integration import run_command as conda_inprocess +from .channel_testing.helpers import http_server_auth_basic # noqa: F401 +from .channel_testing.helpers import http_server_auth_basic_email # noqa: F401 +from .channel_testing.helpers import http_server_auth_none # noqa: F401 +from .channel_testing.helpers import http_server_auth_token # noqa: F401 +from .channel_testing.helpers import create_with_channel from .utils import conda_subprocess, write_env_config +DATA = Path(__file__).parent / "data" + def test_channel_matchspec(): stdout, *_ = conda_inprocess( @@ -84,9 +91,19 @@ def test_channels_installed_unavailable(): assert retcode == 0 -def _setup_channels_alias(prefix): +def _setup_conda_forge_as_defaults(prefix, force=False): write_env_config( prefix, + force=force, + channels=["defaults"], + default_channels=["conda-forge"], + ) + + +def _setup_channels_alias(prefix, force=False): + write_env_config( + prefix, + force=force, channels=["conda-forge", "defaults"], channel_alias="https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud", migrated_channel_aliases=["https://conda.anaconda.org"], @@ -98,9 +115,10 @@ def _setup_channels_alias(prefix): ) -def _setup_channels_custom(prefix): +def _setup_channels_custom(prefix, force=False): write_env_config( prefix, + force=force, channels=["conda-forge", "defaults"], custom_channels={ "conda-forge": "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud", @@ -108,10 +126,6 @@ def _setup_channels_custom(prefix): ) -@pytest.mark.skipif( - datetime.now() < datetime(2023, 6, 15), - reason="Skip until 2023-06-15; remote server has been flaky lately", -) @pytest.mark.parametrize( "config_env", ( @@ -216,3 +230,71 @@ def test_encoding_file_paths(tmp_path: Path): print(process.stderr, file=sys.stderr) assert process.returncode == 0 assert list((tmp_path / "env" / "conda-meta").glob("test-package-*.json")) + + +def test_conda_build_with_aliased_channels(tmp_path): + "https://github.com/conda/conda-libmamba-solver/issues/363" + condarc = Path.home() / ".condarc" + condarc_contents = condarc.read_text() if condarc.is_file() else None + env = os.environ.copy() + if on_win: + env["CONDA_BLD_PATH"] = str(Path(os.environ.get("RUNNER_TEMP", tmp_path), "bld")) + else: + env["CONDA_BLD_PATH"] = str(tmp_path / "conda-bld") + try: + _setup_conda_forge_as_defaults(Path.home(), force=True) + conda_subprocess( + "build", + DATA / "conda_build_recipes" / "jedi", + "--override-channels", + "--channel=defaults", + capture_output=False, + env=env, + ) + finally: + if condarc_contents: + condarc.write_text(condarc_contents) + else: + condarc.unlink() + + +def test_http_server_auth_none(http_server_auth_none): + create_with_channel(http_server_auth_none) + + +def test_http_server_auth_basic(http_server_auth_basic): + create_with_channel(http_server_auth_basic) + + +def test_http_server_auth_basic_email(http_server_auth_basic_email): + create_with_channel(http_server_auth_basic_email) + + +def test_http_server_auth_token(http_server_auth_token): + create_with_channel(http_server_auth_token) + + +def test_http_server_auth_token_in_defaults(http_server_auth_token): + condarc = Path.home() / ".condarc" + condarc_contents = condarc.read_text() if condarc.is_file() else None + try: + write_env_config( + Path.home(), + force=True, + channels=["defaults"], + default_channels=[http_server_auth_token], + ) + reset_context() + conda_subprocess("info", capture_output=False) + conda_subprocess( + "create", + "-p", + _get_temp_prefix(use_restricted_unicode=on_win), + "--solver=libmamba", + "test-package", + ) + finally: + if condarc_contents: + condarc.write_text(condarc_contents) + else: + condarc.unlink() diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 959d44be..b97a4d51 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -11,27 +11,30 @@ DATA = Path(__file__).parent / "data" -def test_build_recipes(): +@pytest.mark.parametrize( + "recipe", + [ + pytest.param(x, id=x.name) + for x in sorted((DATA / "conda_build_recipes").iterdir()) + if (x / "meta.yaml").is_file() + ], +) +def test_build_recipe(recipe): """ Adapted from https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6 See /tests/data/conda_build_recipes/LICENSE for more details """ - recipes_dir = DATA / "conda_build_recipes" - - recipes = [str(x) for x in recipes_dir.iterdir() if x.is_dir()] + expected_fail_recipes = ["baddeps"] env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" - expected_fail_recipes = ["baddeps"] - for recipe in recipes: - recipe_name = Path(recipe).name - print(f"Running {recipe_name}") - if recipe_name in expected_fail_recipes: - with pytest.raises(CalledProcessError): - check_call(["conda-build", recipe], env=env) - else: + recipe_name = Path(recipe).name + if recipe_name in expected_fail_recipes: + with pytest.raises(CalledProcessError): check_call(["conda-build", recipe], env=env) + else: + check_call(["conda-build", recipe], env=env) def test_conda_lock(tmp_path): diff --git a/tests/test_modified_upstream.py b/tests/test_modified_upstream.py deleted file mode 100644 index ea55239c..00000000 --- a/tests/test_modified_upstream.py +++ /dev/null @@ -1,1382 +0,0 @@ -# Copyright (C) 2022 Anaconda, Inc -# Copyright (C) 2023 conda -# SPDX-License-Identifier: BSD-3-Clause -""" -This module fixes some tests found across conda/conda's suite to -check the "spirit" of the test, instead of making explicit comparisons -in stdout messages, overly strict solver checks and other differences -that do not result in incompatible behavior. - -We are copying those offending tests instead of patching them to keep -conda/conda code base as unaffected by this work as possible, but it is -indeed feasible to upgrade those tests in the future for more flexible -comparisons. This is only a workaround during the experimental phase. - -Tests were brought over and patched on Feb 7th, 2022, following the -source found in commit 98fb262c610e17a7731b9183bf37cca98dcc1a71. -""" - -import os -import sys -import warnings -from pprint import pprint - -import pytest -from conda.auxlib.ish import dals -from conda.base.constants import UpdateModifier, on_win -from conda.base.context import conda_tests_ctxt_mgmt_def_pol, context -from conda.common.io import env_var -from conda.core.package_cache_data import PackageCacheData -from conda.core.prefix_data import PrefixData -from conda.core.subdir_data import SubdirData -from conda.exceptions import UnsatisfiableError -from conda.gateways.subprocess import subprocess_call_with_clean_env -from conda.misc import explicit -from conda.models.match_spec import MatchSpec -from conda.models.version import VersionOrder -from conda.testing import ( - CondaCLIFixture, - TmpEnvFixture, - conda_cli, - path_factory, - tmp_env, -) -from conda.testing.cases import BaseTestCase -from conda.testing.helpers import ( - add_subdir, - add_subdir_to_iter, - convert_to_dist_str, - get_solver, - get_solver_2, - get_solver_4, - get_solver_aggregate_1, - get_solver_aggregate_2, - get_solver_cuda, -) -from conda.testing.integration import ( - PYTHON_BINARY, - Commands, - make_temp_env, - package_is_installed, - run_command, -) -from pytest import MonkeyPatch -from pytest_mock import MockerFixture - - -@pytest.mark.integration -class PatchedCondaTestCreate(BaseTestCase): - """ - These tests come from `conda/conda::tests/test_create.py` - """ - - def setUp(self): - PackageCacheData.clear() - - @pytest.mark.xfail( ## MODIFIED - reason="This is not allowed in libmamba: " - "https://github.com/conda/conda-libmamba-solver/pull/289" - ) - def test_pinned_override_with_explicit_spec(self): - with make_temp_env("python=3.8") as prefix: - ## MODIFIED - # Original test assumed the `python=3.6` spec above resolves to `python=3.6.5` - # Instead we only pin whatever the solver decided to install - # Original lines were: - ### run_command(Commands.CONFIG, prefix, - ### "--add", "pinned_packages", "python=3.6.5") - python = next(PrefixData(prefix).query("python")) - run_command( - Commands.CONFIG, prefix, "--add", "pinned_packages", f"python={python.version}" - ) - ## /MODIFIED - - run_command(Commands.INSTALL, prefix, "python=3.7", no_capture=True) - assert package_is_installed(prefix, "python=3.7") - - @pytest.mark.xfail(on_win, reason="TODO: Investigate why this fails on Windows only") - def test_install_update_deps_only_deps_flags(self): - with make_temp_env("flask=2.0.1", "jinja2=3.0.1") as prefix: - python = os.path.join(prefix, PYTHON_BINARY) - result_before = subprocess_call_with_clean_env([python, "--version"]) - assert package_is_installed(prefix, "flask=2.0.1") - assert package_is_installed(prefix, "jinja2=3.0.1") - run_command( - Commands.INSTALL, - prefix, - "flask", - "python", - "--update-deps", - "--only-deps", - no_capture=True, - ) - result_after = subprocess_call_with_clean_env([python, "--version"]) - assert result_before == result_after - assert package_is_installed(prefix, "flask=2.0.1") - assert package_is_installed(prefix, "jinja2>3.0.1") - - -@pytest.mark.xfail(on_win, reason="nomkl not present on windows", strict=True) -def test_install_features(): - # MODIFIED: Added fixture manually - PackageCacheData.clear() - # /MODIFIED - with make_temp_env("python=2", "numpy=1.13", "nomkl", no_capture=True) as prefix: - assert package_is_installed(prefix, "numpy") - assert package_is_installed(prefix, "nomkl") - assert not package_is_installed(prefix, "mkl") - - with make_temp_env("python=2", "numpy=1.13") as prefix: - assert package_is_installed(prefix, "numpy") - assert not package_is_installed(prefix, "nomkl") - assert package_is_installed(prefix, "mkl") - - # run_command(Commands.INSTALL, prefix, "nomkl", no_capture=True) - run_command(Commands.INSTALL, prefix, "python=2", "nomkl", no_capture=True) - # MODIFIED ^: python=2 needed explicitly to trigger update - assert package_is_installed(prefix, "numpy") - assert package_is_installed(prefix, "nomkl") - assert package_is_installed(prefix, "blas=1.0=openblas") - assert not package_is_installed(prefix, "mkl_fft") - assert not package_is_installed(prefix, "mkl_random") - # assert not package_is_installed(prefix, "mkl") # pruned as an indirect dep - - -# The following tests come from `conda/conda::tests/core/test_solve.py` - - -@pytest.mark.integration -def test_pinned_1(tmpdir): - specs = (MatchSpec("numpy"),) - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-3.3.2-0", - "channel-1::numpy-1.7.1-py33_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - with env_var( - "CONDA_PINNED_PACKAGES", - "python=2.6&iopro<=1.4.2", - stack_callback=conda_tests_ctxt_mgmt_def_pol, - ): - specs = (MatchSpec("system=5.8=0"),) - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter(("channel-1::system-5.8-0",)) - assert convert_to_dist_str(final_state_1) == order - - # ignore_pinned=True - specs_to_add = (MatchSpec("python"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state(ignore_pinned=True) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-3.3.2-0", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - # ignore_pinned=False - specs_to_add = (MatchSpec("python"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state(ignore_pinned=False) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-2.6.8-6", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - # incompatible CLI and configured specs - specs_to_add = (MatchSpec("scikit-learn==0.13"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - ## MODIFIED - # Original tests checks for SpecsConfigurationConflictError being raised - # but libmamba will fails with UnsatisfiableError instead. Hence, we check - # the error string. Original check inspected the kwargs of the exception: - ### with pytest.raises(SpecsConfigurationConflictError) as exc: - ### solver.solve_final_state(ignore_pinned=False) - ### kwargs = exc.value._kwargs - ### assert kwargs["requested_specs"] == ["scikit-learn==0.13"] - ### assert kwargs["pinned_specs"] == ["python=2.6"] - with pytest.raises(UnsatisfiableError) as exc_info: - solver.solve_final_state(ignore_pinned=False) - error = str(exc_info.value) - assert "package scikit-learn-0.13" in error - assert "requires python 2.7*" in error - ## /MODIFIED - - specs_to_add = (MatchSpec("numba"),) - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_2, - history_specs=history_specs, - ) as solver: - final_state_3 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_3)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::argparse-1.2.1-py26_0", - "channel-1::llvmpy-0.11.2-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.8.1-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_3) == order - - specs_to_add = (MatchSpec("python"),) - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - MatchSpec("numba"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_3, - history_specs=history_specs, - ) as solver: - final_state_4 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_DEPS) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_4)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::argparse-1.2.1-py26_0", - "channel-1::llvmpy-0.11.2-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.8.1-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_4) == order - - specs_to_add = (MatchSpec("python"),) - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - MatchSpec("numba"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_4, - history_specs=history_specs, - ) as solver: - final_state_5 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_ALL) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_5)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::argparse-1.2.1-py26_0", - "channel-1::llvmpy-0.11.2-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.8.1-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_5) == order - - # now update without pinning - # MODIFIED: libmamba decides to stay in python=2.6 unless explicit - # specs_to_add = (MatchSpec("python"),) - specs_to_add = (MatchSpec("python=3"),) - # /MODIFIED - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - MatchSpec("numba"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_4, - history_specs=history_specs, - ) as solver: - final_state_5 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_ALL) - # PrefixDag(final_state_1, specs).open_url() - print(convert_to_dist_str(final_state_5)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-3.3.2-0", - "channel-1::llvmpy-0.11.2-py33_0", - "channel-1::numpy-1.7.1-py33_0", - "channel-1::numba-0.8.1-np17py33_0", - ) - ) - assert convert_to_dist_str(final_state_5) == order - - -@pytest.mark.integration -def test_freeze_deps_1(tmpdir): - specs = (MatchSpec("six=1.7"),) - with get_solver_2(tmpdir, specs) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-2::openssl-1.0.2l-0", - "channel-2::readline-6.2-2", - "channel-2::sqlite-3.13.0-0", - "channel-2::tk-8.5.18-0", - "channel-2::xz-5.2.3-0", - "channel-2::zlib-1.2.11-0", - "channel-2::python-3.4.5-0", - "channel-2::six-1.7.3-py34_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - specs_to_add = (MatchSpec("bokeh"),) - with get_solver_2( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = () - link_order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::backports_abc-0.5-py34_0", - "channel-2::markupsafe-1.0-py34_0", - "channel-2::numpy-1.13.0-py34_0", - "channel-2::pyyaml-3.12-py34_0", - "channel-2::requests-2.14.2-py34_0", - "channel-2::setuptools-27.2.0-py34_0", - "channel-2::jinja2-2.9.6-py34_0", - "channel-2::python-dateutil-2.6.1-py34_0", - "channel-2::tornado-4.4.2-py34_0", - "channel-2::bokeh-0.12.4-py34_0", - ) - ) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - # now we can't install the latest bokeh 0.12.5, but instead we get bokeh 0.12.4 - specs_to_add = (MatchSpec("bokeh"),) - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = () - link_order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::backports_abc-0.5-py34_0", - "channel-2::markupsafe-1.0-py34_0", - "channel-2::numpy-1.13.0-py34_0", - "channel-2::pyyaml-3.12-py34_0", - "channel-2::requests-2.14.2-py34_0", - "channel-2::setuptools-27.2.0-py34_0", - "channel-2::jinja2-2.9.6-py34_0", - "channel-2::python-dateutil-2.6.1-py34_0", - "channel-2::tornado-4.4.2-py34_0", - "channel-2::bokeh-0.12.4-py34_0", - ) - ) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - # here, the python=3.4 spec can't be satisfied, so it's dropped, and we go back to py27 - with pytest.raises(UnsatisfiableError): - specs_to_add = (MatchSpec("bokeh=0.12.5"),) - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - unlink_precs, link_precs = solver.solve_for_diff() - - # adding the explicit python spec allows conda to change the python versions. - # one possible outcome is that this updates to python 3.6. That is not desirable because of the - # explicit "six=1.7" request in the history. It should only neuter that spec if there's no way - # to solve it with that spec. - specs_to_add = MatchSpec("bokeh=0.12.5"), MatchSpec("python") - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - - unlink_precs, link_precs = solver.solve_for_diff() - - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - "channel-2::six-1.7.3-py34_0", - "channel-2::python-3.4.5-0", - # MODIFIED: xz is not uninstalled for some reason in libmamba :shrug: - # "channel-2::xz-5.2.3-0", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::python-2.7.13-0", - "channel-2::backports-1.0-py27_0", - "channel-2::backports_abc-0.5-py27_0", - "channel-2::certifi-2016.2.28-py27_0", - "channel-2::futures-3.1.1-py27_0", - "channel-2::markupsafe-1.0-py27_0", - "channel-2::numpy-1.13.1-py27_0", - "channel-2::pyyaml-3.12-py27_0", - "channel-2::requests-2.14.2-py27_0", - "channel-2::six-1.7.3-py27_0", - "channel-2::python-dateutil-2.6.1-py27_0", - "channel-2::setuptools-36.4.0-py27_1", - "channel-2::singledispatch-3.4.0.3-py27_0", - "channel-2::ssl_match_hostname-3.5.0.1-py27_0", - "channel-2::jinja2-2.9.6-py27_0", - "channel-2::tornado-4.5.2-py27_0", - "channel-2::bokeh-0.12.5-py27_1", - ) - ) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - # here, the python=3.4 spec can't be satisfied, so it's dropped, and we go back to py27 - specs_to_add = (MatchSpec("bokeh=0.12.5"),) - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - with pytest.raises(UnsatisfiableError): - ## ADDED - solver._command = "install" - ## /ADDED - solver.solve_final_state(update_modifier=UpdateModifier.FREEZE_INSTALLED) - - -def test_cuda_fail_1(tmpdir): - specs = (MatchSpec("cudatoolkit"),) - - # No cudatoolkit in index for CUDA 8.0 - with env_var("CONDA_OVERRIDE_CUDA", "8.0"): - with get_solver_cuda(tmpdir, specs) as solver: - with pytest.raises(UnsatisfiableError) as exc: - final_state = solver.solve_final_state() - - ## MODIFIED - # libmamba will generate a slightly different error message, but the spirit is the same. - # Original check was: - ### if sys.platform == "darwin": - ### plat = "osx-64" - ### elif sys.platform == "linux": - ### plat = "linux-64" - ### elif sys.platform == "win32": - ### if platform.architecture()[0] == "32bit": - ### plat = "win-32" - ### else: - ### plat = "win-64" - ### else: - ### plat = "linux-64" - ### assert str(exc.value).strip() == dals("""The following specifications were found to be incompatible with your system: - ### - ### - feature:/{}::__cuda==8.0=0 - ### - cudatoolkit -> __cuda[version='>=10.0|>=9.0'] - ### - ### Your installed version is: 8.0""".format(plat)) - possible_messages = [ - dals( - """Encountered problems while solving: - - nothing provides __cuda >=9.0 needed by cudatoolkit-9.0-0""" - ), - dals( - """Encountered problems while solving: - - nothing provides __cuda >=10.0 needed by cudatoolkit-10.0-0""" - ), - ] - exc_msg = str(exc.value).strip() - assert any(msg in exc_msg for msg in possible_messages) - ## /MODIFIED - - -def test_cuda_fail_2(tmpdir): - specs = (MatchSpec("cudatoolkit"),) - - # No CUDA on system - with env_var("CONDA_OVERRIDE_CUDA", ""): - with get_solver_cuda(tmpdir, specs) as solver: - with pytest.raises(UnsatisfiableError) as exc: - final_state = solver.solve_final_state() - - ## MODIFIED - # libmamba will generate a slightly different error message, but the spirit is the same. - # Original check was: - ### assert str(exc.value).strip() == dals("""The following specifications were found to be incompatible with your system: - ### - ### - cudatoolkit -> __cuda[version='>=10.0|>=9.0'] - ### - ### Your installed version is: not available""") - possible_messages = [ - dals( - """Encountered problems while solving: - - nothing provides __cuda >=9.0 needed by cudatoolkit-9.0-0""" - ), - dals( - """Encountered problems while solving: - - nothing provides __cuda >=10.0 needed by cudatoolkit-10.0-0""" - ), - ] - exc_msg = str(exc.value).strip() - assert any(msg in exc_msg for msg in possible_messages) - ## /MODIFIED - - -def test_update_all_1(tmpdir): - ## MODIFIED - # Libmamba requires MatchSpec.conda_build_form() internally, which depends on `version` and - # `build` fields. `system` below is using only `build_number`, so we have to adapt the syntax - # accordingly. It should be the same result, but in a conda_build_form-friendly way: - ### specs = MatchSpec("numpy=1.5"), MatchSpec("python=2.6"), MatchSpec("system[build_number=0]") - specs = ( - MatchSpec("numpy=1.5"), - MatchSpec("python=2.6"), - MatchSpec("system[version=*,build=*0]"), - ) - ## /MODIFIED - - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - print(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-2.6.8-6", - "channel-1::numpy-1.5.1-py26_4", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - specs_to_add = MatchSpec("numba=0.6"), MatchSpec("numpy") - with get_solver( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state() - # PrefixDag(final_state_2, specs).open_url() - print(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::llvmpy-0.10.2-py26_0", - "channel-1::nose-1.3.0-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.6.0-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - specs_to_add = (MatchSpec("numba=0.6"),) - with get_solver( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_ALL) - # PrefixDag(final_state_2, specs).open_url() - print(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", # stick with python=2.6 even though UPDATE_ALL - "channel-1::llvmpy-0.10.2-py26_0", - "channel-1::nose-1.3.0-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.6.0-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - -def test_conda_downgrade(tmpdir): - specs = (MatchSpec("conda-build"),) - with env_var("CONDA_CHANNEL_PRIORITY", "False", stack_callback=conda_tests_ctxt_mgmt_def_pol): - with get_solver_aggregate_1(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-2::conda-env-2.6.0-0", - "channel-2::libffi-3.2.1-1", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-2::zlib-1.2.11-0", - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::patchelf-0.9-hf484d3e_2", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::asn1crypto-0.24.0-py37_0", - "channel-4::beautifulsoup4-4.6.3-py37_0", - "channel-4::certifi-2018.8.13-py37_0", - "channel-4::chardet-3.0.4-py37_1", - "channel-4::cryptography-vectors-2.3-py37_0", - "channel-4::filelock-3.0.4-py37_0", - "channel-4::glob2-0.6-py37_0", - "channel-4::idna-2.7-py37_0", - "channel-4::markupsafe-1.0-py37h14c3975_1", - "channel-4::pkginfo-1.4.2-py37_1", - "channel-4::psutil-5.4.6-py37h14c3975_0", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - "channel-4::pycparser-2.18-py37_1", - "channel-4::pysocks-1.6.8-py37_0", - "channel-4::pyyaml-3.13-py37h14c3975_0", - "channel-4::ruamel_yaml-0.15.46-py37h14c3975_0", - "channel-4::six-1.11.0-py37_1", - "channel-4::cffi-1.11.5-py37h9745a5d_0", - "channel-4::setuptools-40.0.0-py37_0", - "channel-4::cryptography-2.3-py37hb7f436b_0", - "channel-4::jinja2-2.10-py37_0", - "channel-4::pyopenssl-18.0.0-py37_0", - "channel-4::urllib3-1.23-py37_0", - "channel-4::requests-2.19.1-py37_0", - "channel-4::conda-4.5.10-py37_0", - "channel-4::conda-build-3.12.1-py37_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - SubdirData.clear_cached_local_channel_data() - specs_to_add = (MatchSpec("itsdangerous"),) # MatchSpec("conda"), - saved_sys_prefix = sys.prefix - try: - sys.prefix = tmpdir.strpath - with get_solver_aggregate_1( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = ( - # no conda downgrade - ) - link_order = add_subdir_to_iter(("channel-2::itsdangerous-0.24-py_0",)) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - specs_to_add = ( - MatchSpec("itsdangerous"), - MatchSpec("conda"), - ) - with get_solver_aggregate_1( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - specs_to_add = MatchSpec("itsdangerous"), MatchSpec("conda<4.4.10"), MatchSpec("python") - with get_solver_aggregate_1( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - # now conda gets downgraded - "channel-4::conda-build-3.12.1-py37_0", - "channel-4::conda-4.5.10-py37_0", - "channel-4::requests-2.19.1-py37_0", - "channel-4::urllib3-1.23-py37_0", - "channel-4::pyopenssl-18.0.0-py37_0", - "channel-4::jinja2-2.10-py37_0", - "channel-4::cryptography-2.3-py37hb7f436b_0", - "channel-4::setuptools-40.0.0-py37_0", - "channel-4::cffi-1.11.5-py37h9745a5d_0", - "channel-4::six-1.11.0-py37_1", - "channel-4::ruamel_yaml-0.15.46-py37h14c3975_0", - "channel-4::pyyaml-3.13-py37h14c3975_0", - "channel-4::pysocks-1.6.8-py37_0", - "channel-4::pycparser-2.18-py37_1", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - "channel-4::psutil-5.4.6-py37h14c3975_0", - "channel-4::pkginfo-1.4.2-py37_1", - "channel-4::markupsafe-1.0-py37h14c3975_1", - "channel-4::idna-2.7-py37_0", - "channel-4::glob2-0.6-py37_0", - "channel-4::filelock-3.0.4-py37_0", - "channel-4::cryptography-vectors-2.3-py37_0", - "channel-4::chardet-3.0.4-py37_1", - "channel-4::certifi-2018.8.13-py37_0", - "channel-4::beautifulsoup4-4.6.3-py37_0", - "channel-4::asn1crypto-0.24.0-py37_0", - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::ncurses-6.1-hf484d3e_0", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-2::openssl-1.0.2l-0", - "channel-2::readline-6.2-2", - "channel-2::sqlite-3.13.0-0", - "channel-2::tk-8.5.18-0", - "channel-2::xz-5.2.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::python-3.6.2-0", - "channel-2::asn1crypto-0.22.0-py36_0", - "channel-4::beautifulsoup4-4.6.3-py36_0", - "channel-2::certifi-2016.2.28-py36_0", - "channel-4::chardet-3.0.4-py36_1", - "channel-4::filelock-3.0.4-py36_0", - "channel-4::glob2-0.6-py36_0", - "channel-2::idna-2.6-py36_0", - "channel-2::itsdangerous-0.24-py36_0", - "channel-2::markupsafe-1.0-py36_0", - "channel-4::pkginfo-1.4.2-py36_1", - "channel-2::psutil-5.2.2-py36_0", - "channel-2::pycosat-0.6.2-py36_0", - "channel-2::pycparser-2.18-py36_0", - "channel-2::pyparsing-2.2.0-py36_0", - "channel-2::pyyaml-3.12-py36_0", - "channel-2::requests-2.14.2-py36_0", - "channel-2::ruamel_yaml-0.11.14-py36_1", - "channel-2::six-1.10.0-py36_0", - "channel-2::cffi-1.10.0-py36_0", - "channel-2::packaging-16.8-py36_0", - "channel-2::setuptools-36.4.0-py36_1", - "channel-2::cryptography-1.8.1-py36_0", - "channel-2::jinja2-2.9.6-py36_0", - "channel-2::pyopenssl-17.0.0-py36_0", - "channel-2::conda-4.3.30-py36h5d9f9f4_0", - "channel-4::conda-build-3.12.1-py36_0", - ) - ) - ## MODIFIED - # Original checks verified the full solution was strictly matched: - ### assert convert_to_dist_str(unlink_precs) == unlink_order - ### assert convert_to_dist_str(link_precs) == link_order - # We only check for conda itself and the explicit specs - # The other packages are slightly different; - # again libedit and ncurses are involved - # (they are also involved in test_fast_update_with_update_modifier_not_set) - for pkg in link_precs: - if pkg.name == "conda": - assert VersionOrder(pkg.version) < VersionOrder("4.4.10") - # TODO: these assertions are a bit flaky (only true in some attempts) - # to be fixed at https://github.com/conda/conda-libmamba-solver/issues/317 - # elif pkg.name == "python": - # assert pkg.version == "3.6.2" - # elif pkg.name == "conda-build": - # assert pkg.version == "3.12.1" - # elif pkg.name == "itsdangerous": - # assert pkg.version == "0.24" - ## /MODIFIED - finally: - sys.prefix = saved_sys_prefix - - -def test_python2_update(tmpdir): - # Here we're actually testing that a user-request will uninstall incompatible packages - # as necessary. - specs = MatchSpec("conda"), MatchSpec("python=2") - with get_solver_4(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order1 = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-4::conda-env-2.6.0-1", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-4::libffi-3.2.1-hd88cf55_4", - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::zlib-1.2.11-ha838bed_2", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::python-2.7.15-h1571d57_0", - "channel-4::asn1crypto-0.24.0-py27_0", - "channel-4::certifi-2018.8.13-py27_0", - "channel-4::chardet-3.0.4-py27_1", - "channel-4::cryptography-vectors-2.3-py27_0", - "channel-4::enum34-1.1.6-py27_1", - "channel-4::futures-3.2.0-py27_0", - "channel-4::idna-2.7-py27_0", - "channel-4::ipaddress-1.0.22-py27_0", - "channel-4::pycosat-0.6.3-py27h14c3975_0", - "channel-4::pycparser-2.18-py27_1", - "channel-4::pysocks-1.6.8-py27_0", - "channel-4::ruamel_yaml-0.15.46-py27h14c3975_0", - "channel-4::six-1.11.0-py27_1", - "channel-4::cffi-1.11.5-py27h9745a5d_0", - "channel-4::cryptography-2.3-py27hb7f436b_0", - "channel-4::pyopenssl-18.0.0-py27_0", - "channel-4::urllib3-1.23-py27_0", - "channel-4::requests-2.19.1-py27_0", - "channel-4::conda-4.5.10-py27_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order1 - - specs_to_add = (MatchSpec("python=3"),) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-4::conda-env-2.6.0-1", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-4::libffi-3.2.1-hd88cf55_4", - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::zlib-1.2.11-ha838bed_2", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::asn1crypto-0.24.0-py37_0", - "channel-4::certifi-2018.8.13-py37_0", - "channel-4::chardet-3.0.4-py37_1", - "channel-4::idna-2.7-py37_0", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - "channel-4::pycparser-2.18-py37_1", - "channel-4::pysocks-1.6.8-py37_0", - "channel-4::ruamel_yaml-0.15.46-py37h14c3975_0", - "channel-4::six-1.11.0-py37_1", - "channel-4::cffi-1.11.5-py37h9745a5d_0", - "channel-4::cryptography-2.2.2-py37h14c3975_0", - "channel-4::pyopenssl-18.0.0-py37_0", - "channel-4::urllib3-1.23-py37_0", - "channel-4::requests-2.19.1-py37_0", - "channel-4::conda-4.5.10-py37_0", - ) - ) - - ## MODIFIED - # libmamba has a different solution here (cryptography 2.3 instead of 2.2.2) - # and cryptography-vectors (not present in regular conda) - # they are essentially the same functional solution; the important part here - # is that the env migrated to Python 3.7, so we only check some packages - # Original check: - ### assert convert_to_dist_str(final_state_2) == order - full_solution = convert_to_dist_str(final_state_2) - important_parts = add_subdir_to_iter( - ( - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::conda-4.5.10-py37_0", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - ) - ) - assert set(important_parts).issubset(set(full_solution)) - ## /MODIFIED - - -def test_fast_update_with_update_modifier_not_set(tmpdir): - specs = ( - MatchSpec("python=2"), - MatchSpec("openssl==1.0.2l"), - MatchSpec("sqlite=3.21"), - ) - with get_solver_4(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order1 = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-4::libffi-3.2.1-hd88cf55_4", - "channel-4::ncurses-6.0-h9df7e31_2", - "channel-4::openssl-1.0.2l-h077ae2c_5", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::zlib-1.2.11-ha838bed_2", - "channel-4::libedit-3.1-heed3624_0", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.21.0-h1bed415_2", - "channel-4::python-2.7.14-h89e7a4a_22", - ) - ) - assert convert_to_dist_str(final_state_1) == order1 - - specs_to_add = (MatchSpec("python"),) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - "channel-4::python-2.7.14-h89e7a4a_22", - "channel-4::libedit-3.1-heed3624_0", - "channel-4::openssl-1.0.2l-h077ae2c_5", - "channel-4::ncurses-6.0-h9df7e31_2", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::python-3.6.4-hc3d631a_1", # python is upgraded - ) - ) - ## MODIFIED - # We only check python was upgraded as expected, not the full solution - ### assert convert_to_dist_str(unlink_precs) == unlink_order - ### assert convert_to_dist_str(link_precs) == link_order - assert add_subdir("channel-4::python-2.7.14-h89e7a4a_22") in convert_to_dist_str( - unlink_precs - ) - assert add_subdir("channel-4::python-3.6.4-hc3d631a_1") in convert_to_dist_str(link_precs) - ## /MODIFIED - - specs_to_add = (MatchSpec("sqlite"),) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - "channel-4::python-2.7.14-h89e7a4a_22", - "channel-4::sqlite-3.21.0-h1bed415_2", - "channel-4::libedit-3.1-heed3624_0", - "channel-4::openssl-1.0.2l-h077ae2c_5", - "channel-4::ncurses-6.0-h9df7e31_2", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::sqlite-3.24.0-h84994c4_0", # sqlite is upgraded - "channel-4::python-2.7.15-h1571d57_0", # python is not upgraded - ) - ) - ## MODIFIED - # We only check sqlite was upgraded as expected and python stays the same - ### assert convert_to_dist_str(unlink_precs) == unlink_order - ### assert convert_to_dist_str(link_precs) == link_order - assert add_subdir("channel-4::sqlite-3.21.0-h1bed415_2") in convert_to_dist_str( - unlink_precs - ) - sqlite = next(pkg for pkg in link_precs if pkg.name == "sqlite") - # mamba chooses a different sqlite version (3.23 instead of 3.24) - assert VersionOrder(sqlite.version) > VersionOrder("3.21") - # If Python was changed, it should have stayed at 2.7 - python = next((pkg for pkg in link_precs if pkg.name == "python"), None) - if python: - assert python.version.startswith("2.7") - ## /MODIFIED - - specs_to_add = ( - MatchSpec("sqlite"), - MatchSpec("python"), - ) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state( - update_modifier=UpdateModifier.SPECS_SATISFIED_SKIP_SOLVE - ) - pprint(convert_to_dist_str(final_state_2)) - assert convert_to_dist_str(final_state_2) == order1 - - -@pytest.mark.xfail(True, reason="Known bug: mamba prefers arch to noarch - TODO") -def test_channel_priority_churn_minimized(tmpdir): - specs = ( - MatchSpec("conda-build"), - MatchSpec("itsdangerous"), - ) - with get_solver_aggregate_2(tmpdir, specs) as solver: - final_state = solver.solve_final_state() - - pprint(convert_to_dist_str(final_state)) - - with get_solver_aggregate_2( - tmpdir, [MatchSpec("itsdangerous")], prefix_records=final_state, history_specs=specs - ) as solver: - solver.channels.reverse() - unlink_dists, link_dists = solver.solve_for_diff( - update_modifier=UpdateModifier.FREEZE_INSTALLED - ) - pprint(convert_to_dist_str(unlink_dists)) - pprint(convert_to_dist_str(link_dists)) - assert len(unlink_dists) == 1 - assert len(link_dists) == 1 - - -@pytest.mark.xfail(True, reason="channel priority is a bit different in libmamba; TODO") -def test_priority_1(tmpdir): - with env_var("CONDA_SUBDIR", "linux-64", stack_callback=conda_tests_ctxt_mgmt_def_pol): - specs = ( - MatchSpec("pandas"), - MatchSpec("python=2.7"), - ) - - ## MODIFIED - # Original value was set to True (legacy value for "flexible" nowadays), but libmamba - # only gets the same solution is strict priority is chosen. It _looks_ like this was the - # intention of the test anyways, but it should be investigated further. Marking as xfail for now. - ### with env_var("CONDA_CHANNEL_PRIORITY", "True", stack_callback=conda_tests_ctxt_mgmt_def_pol): - with env_var( - "CONDA_CHANNEL_PRIORITY", "strict", stack_callback=conda_tests_ctxt_mgmt_def_pol - ): - ## /MODIFIED - - with get_solver_aggregate_1(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::openssl-1.0.2l-0", - "channel-2::readline-6.2-2", - "channel-2::sqlite-3.13.0-0", - "channel-2::tk-8.5.18-0", - "channel-2::zlib-1.2.11-0", - "channel-2::python-2.7.13-0", - "channel-2::numpy-1.13.1-py27_0", - "channel-2::pytz-2017.2-py27_0", - "channel-2::six-1.10.0-py27_0", - "channel-2::python-dateutil-2.6.1-py27_0", - "channel-2::pandas-0.20.3-py27_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - with env_var( - "CONDA_CHANNEL_PRIORITY", "False", stack_callback=conda_tests_ctxt_mgmt_def_pol - ): - with get_solver_aggregate_1( - tmpdir, specs, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_2)) - # python and pandas will be updated as they are explicit specs. Other stuff may or may not, - # as required to satisfy python and pandas - order = add_subdir_to_iter( - ( - "channel-4::python-2.7.15-h1571d57_0", - "channel-4::pandas-0.23.4-py27h04863e7_0", - ) - ) - for spec in order: - assert spec in convert_to_dist_str(final_state_2) - - # channel priority taking effect here. channel-2 should be the channel to draw from. Downgrades expected. - # python and pandas will be updated as they are explicit specs. Other stuff may or may not, - # as required to satisfy python and pandas - with get_solver_aggregate_1( - tmpdir, specs, prefix_records=final_state_2, history_specs=specs - ) as solver: - final_state_3 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_3)) - order = add_subdir_to_iter( - ( - "channel-2::python-2.7.13-0", - "channel-2::pandas-0.20.3-py27_0", - ) - ) - for spec in order: - assert spec in convert_to_dist_str(final_state_3) - - specs_to_add = (MatchSpec("six<1.10"),) - specs_to_remove = (MatchSpec("pytz"),) - with get_solver_aggregate_1( - tmpdir, - specs_to_add=specs_to_add, - specs_to_remove=specs_to_remove, - prefix_records=final_state_3, - history_specs=specs, - ) as solver: - final_state_4 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_4)) - order = add_subdir_to_iter( - ( - "channel-2::python-2.7.13-0", - "channel-2::six-1.9.0-py27_0", - ) - ) - for spec in order: - assert spec in convert_to_dist_str(final_state_4) - assert "pandas" not in convert_to_dist_str(final_state_4) - - -def test_downgrade_python_prevented_with_sane_message(tmpdir): - specs = (MatchSpec("python=2.6"),) - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-2.6.8-6", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - # incompatible CLI and configured specs - specs_to_add = (MatchSpec("scikit-learn==0.13"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - with pytest.raises(UnsatisfiableError) as exc: - solver.solve_final_state() - - error_msg = str(exc.value).strip() - - ## MODIFIED - # One more case of different wording for the same message. I think the essence is the same - # (cannot update to python 2.7), even if python 2.6 is not mentioned. - ### assert "incompatible with the existing python installation in your environment:" in error_msg - ### assert "- scikit-learn==0.13 -> python=2.7" in error_msg - ### assert "Your python: python=2.6" in error_msg - assert "Encountered problems while solving" in error_msg - assert "package scikit-learn-0.13" in error_msg and "requires python 2.7*" in error_msg - ## /MODIFIED - - specs_to_add = (MatchSpec("unsatisfiable-with-py26"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - with pytest.raises(UnsatisfiableError) as exc: - solver.solve_final_state() - error_msg = str(exc.value).strip() - - ## MODIFIED - # In this case, the error is not as similar! We are still accepting it, but it could use - # some improvements... Note how Python is not mentioned at all, just scikit-learn. - # Leaving a # TODO mark here so we can come revisit this in the future. - ### assert "incompatible with the existing python installation in your environment:" in error_msg - ### assert "- unsatisfiable-with-py26 -> python=2.7" in error_msg - ### assert "Your python: python=2.6" - assert "Encountered problems while solving" in error_msg - assert "package unsatisfiable-with-py26-1.0-0 requires scikit-learn 0.13" in error_msg - ## /MODIFIED - - -# The following tests come from tests/test_priority.py - - -@pytest.mark.integration -@pytest.mark.parametrize( - "pinned_package", - [ - pytest.param(True, id="with pinned_package"), - pytest.param(False, id="without pinned_package"), - ], -) -def test_reorder_channel_priority( - tmp_env: TmpEnvFixture, - monkeypatch: MonkeyPatch, - conda_cli: CondaCLIFixture, - pinned_package: bool, -): - # use "cheap" packages with no dependencies - package1 = "zlib" - package2 = "ca-certificates" - - # set pinned package - if pinned_package: - monkeypatch.setenv("CONDA_PINNED_PACKAGES", package1) - - # create environment with package1 and package2 - with tmp_env("--override-channels", "--channel=defaults", package1, package2) as prefix: - # check both packages are installed from defaults - PrefixData._cache_.clear() - assert PrefixData(prefix).get(package1).channel.name == "pkgs/main" - assert PrefixData(prefix).get(package2).channel.name == "pkgs/main" - - # update --all - out, err, retcode = conda_cli( - "update", - f"--prefix={prefix}", - "--override-channels", - "--channel=conda-forge", - "--all", - "--yes", - ) - # check pinned package is unchanged but unpinned packages are updated from conda-forge - PrefixData._cache_.clear() - expected_channel = "pkgs/main" if pinned_package else "conda-forge" - assert PrefixData(prefix).get(package1).channel.name == expected_channel - # assert PrefixData(prefix).get(package2).channel.name == "conda-forge" - # MODIFIED ^: Some packages do not change channels in libmamba - - -def test_explicit_missing_cache_entries( - mocker: MockerFixture, - conda_cli: CondaCLIFixture, - tmp_env: TmpEnvFixture, -): - """Test that explicit() raises and notifies if some of the specs were not found in the cache.""" - from conda.core.package_cache_data import PackageCacheData - - with tmp_env() as prefix: # ensure writable env - if len(PackageCacheData.get_all_extracted_entries()) == 0: - # Package cache e.g. ./devenv/Darwin/x86_64/envs/devenv-3.9-c/pkgs/ can - # be empty in certain cases (Noted in OSX with Python 3.9, when - # Miniconda installs Python 3.10). Install a small package. - warnings.warn("test_explicit_missing_cache_entries: No packages in cache.") - out, err, retcode = conda_cli("install", "--prefix", prefix, "heapdict", "--yes") - assert retcode == 0, (out, err) # MODIFIED - - # Patching ProgressiveFetchExtract prevents trying to download a package from the url. - # Note that we cannot monkeypatch context.dry_run, because explicit() would exit early with that. - mocker.patch("conda.misc.ProgressiveFetchExtract") - print(PackageCacheData.get_all_extracted_entries()[0]) # MODIFIED - with pytest.raises( - AssertionError, - match="Missing package cache records for: pkgs/linux-64::foo==1.0.0=py_0", - ): - explicit( - [ - "http://test/pkgs/linux-64/foo-1.0.0-py_0.tar.bz2", # does not exist - PackageCacheData.get_all_extracted_entries()[0].url, # exists - ], - prefix, - )