From 6a34776841f6a3ffbe2aa6b285aff9cf9d72692d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 4 Aug 2023 10:36:03 +1000 Subject: [PATCH 1/2] Use new Python API when resolving Python Env Vars --- src/platform/api/pythonApi.ts | 39 +++----- src/platform/interpreter/activation/types.ts | 3 +- src/platform/interpreter/contracts.ts | 8 ++ .../environmentActivationService.node.ts | 97 ++++++++++++------- .../globalPythonExePathService.node.ts | 35 ++++--- src/platform/interpreter/helpers.ts | 23 ++++- 6 files changed, 123 insertions(+), 82 deletions(-) diff --git a/src/platform/api/pythonApi.ts b/src/platform/api/pythonApi.ts index f0a3fd90735..cec806c3ea8 100644 --- a/src/platform/api/pythonApi.ts +++ b/src/platform/api/pythonApi.ts @@ -25,6 +25,7 @@ import { PythonExtensionActicationFailedError } from '../errors/pythonExtActivat import { PythonExtensionApiNotExportedError } from '../errors/pythonExtApiNotExportedError'; import { getOSType, OSType } from '../common/utils/platform'; import { SemVer } from 'semver'; +import { getEnvironmentType } from '../interpreter/helpers'; export function deserializePythonEnvironment( pythonVersion: Partial | undefined, @@ -123,31 +124,7 @@ export function resolvedPythonEnvToJupyterEnv( }; } export function pythonEnvToJupyterEnv(env: Environment, supportsEmptyCondaEnv: boolean): PythonEnvironment | undefined { - const envTools = env.tools as KnownEnvironmentTools[]; - // Map the Python env tool to a Jupyter environment type. - const orderOrEnvs: [pythonEnvTool: KnownEnvironmentTools, JupyterEnv: EnvironmentType][] = [ - ['Conda', EnvironmentType.Conda], - ['Pyenv', EnvironmentType.Pyenv], - ['Pipenv', EnvironmentType.Pipenv], - ['Poetry', EnvironmentType.Poetry], - ['VirtualEnvWrapper', EnvironmentType.VirtualEnvWrapper], - ['VirtualEnv', EnvironmentType.VirtualEnv], - ['Venv', EnvironmentType.Venv] - ]; - let envType = envTools.length ? (envTools[0] as EnvironmentType) : EnvironmentType.Unknown; - if (env.environment?.type === 'Conda') { - envType = EnvironmentType.Conda; - } else { - for (const [pythonEnvTool, JupyterEnv] of orderOrEnvs) { - if (envTools.includes(pythonEnvTool)) { - envType = JupyterEnv; - break; - } - } - if (envType === EnvironmentType.Unknown && env.environment?.type === 'VirtualEnvironment') { - envType = EnvironmentType.VirtualEnv; - } - } + const envType = getEnvironmentType(env); let isCondaEnvWithoutPython = false; let uri: Uri; let id = env.id; @@ -429,6 +406,9 @@ export class InterpreterService implements IInterpreterService { private refreshPromises = new PromiseMonitor(); private pauseEnvDetection = false; private readonly onResumeEnvDetection = new EventEmitter(); + public get known() { + return this.api?.environments.known || []; + } constructor( @inject(IPythonApiProvider) private readonly apiProvider: IPythonApiProvider, @inject(IPythonExtensionChecker) private extensionChecker: IPythonExtensionChecker, @@ -465,6 +445,15 @@ export class InterpreterService implements IInterpreterService { this.disposables ); } + public async resolveEnvironment(id: string | Environment): Promise { + return this.getApi().then((api) => { + if (!api) { + return; + } + const env = typeof id === 'string' ? api.environments.known.find((e) => e.id === id || e.path === id) : id; + return api.environments.resolveEnvironment(env || id); + }); + } public get onDidChangeInterpreter(): Event { this.hookupOnDidChangeInterpreterEvent(); return this.didChangeInterpreter.event; diff --git a/src/platform/interpreter/activation/types.ts b/src/platform/interpreter/activation/types.ts index b3938198976..85198c526a1 100644 --- a/src/platform/interpreter/activation/types.ts +++ b/src/platform/interpreter/activation/types.ts @@ -3,13 +3,12 @@ import { CancellationToken } from 'vscode'; import { Resource } from '../../common/types'; -import { PythonEnvironment } from '../../pythonEnvironments/info'; export const IEnvironmentActivationService = Symbol('IEnvironmentActivationService'); export interface IEnvironmentActivationService { getActivatedEnvironmentVariables( resource: Resource, - interpreter: PythonEnvironment, + interpreter: { id: string }, token?: CancellationToken ): Promise; } diff --git a/src/platform/interpreter/contracts.ts b/src/platform/interpreter/contracts.ts index 4f380bd588a..41ed19f7609 100644 --- a/src/platform/interpreter/contracts.ts +++ b/src/platform/interpreter/contracts.ts @@ -3,9 +3,16 @@ import { Event, Uri, CancellationToken } from 'vscode'; import { PythonEnvironment } from '../pythonEnvironments/info'; +import { Environment, ResolvedEnvironment } from '../api/pythonApiTypes'; type InterpreterId = string; export const IInterpreterService = Symbol('IInterpreterService'); export interface IInterpreterService { + // #region New API + readonly known: readonly Environment[]; + resolveEnvironment(id: string | Environment): Promise; + // #endregion + + // #region Old API readonly status: 'refreshing' | 'idle'; readonly onDidChangeStatus: Event; readonly onDidEnvironmentVariablesChange: Event; @@ -39,4 +46,5 @@ export interface IInterpreterService { token?: CancellationToken ): Promise; getInterpreterHash(id: string): string | undefined; + // #endregion } diff --git a/src/platform/interpreter/environmentActivationService.node.ts b/src/platform/interpreter/environmentActivationService.node.ts index 9a6bb843c98..31f3ecb8a54 100644 --- a/src/platform/interpreter/environmentActivationService.node.ts +++ b/src/platform/interpreter/environmentActivationService.node.ts @@ -7,7 +7,7 @@ import * as path from '../vscode-path/path'; import { IWorkspaceService } from '../common/application/types'; import { IDisposable, Resource } from '../common/types'; import { ICustomEnvironmentVariablesProvider, IEnvironmentVariablesService } from '../common/variables/types'; -import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info'; +import { EnvironmentType } from '../pythonEnvironments/info'; import { sendTelemetryEvent } from '../../telemetry'; import { IPythonApiProvider, IPythonExtensionChecker } from '../api/types'; import { StopWatch } from '../common/utils/stopWatch'; @@ -19,11 +19,13 @@ import { KernelProgressReporter } from '../progress/kernelProgressReporter'; import { Telemetry } from '../common/constants'; import { ignoreLogging, logValue, traceDecoratorVerbose, traceError, traceVerbose, traceWarning } from '../logging'; import { TraceOptions } from '../logging/types'; -import { serializePythonEnvironment } from '../api/pythonApi'; +import { pythonEnvToJupyterEnv, serializePythonEnvironment } from '../api/pythonApi'; import { GlobalPythonExecutablePathService } from './globalPythonExePathService.node'; import { noop } from '../common/utils/misc'; import { CancellationToken } from 'vscode'; import { raceCancellation } from '../common/cancellation'; +import { getEnvironmentType, getPythonEnvDisplayName, isCondaEnvironmentWithoutPython } from './helpers'; +import { Environment } from '../api/pythonApiTypes'; const ENV_VAR_CACHE_TIMEOUT = 60_000; @@ -56,28 +58,36 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi } public async getActivatedEnvironmentVariables( resource: Resource, - interpreter: PythonEnvironment, + interpreter: { id: string }, token?: CancellationToken ): Promise { - const title = DataScience.activatingPythonEnvironment( - interpreter.displayName || getDisplayPath(interpreter.uri) - ); - return KernelProgressReporter.wrapAndReportProgress(resource, title, () => - this.getActivatedEnvironmentVariablesImplWithCaching(resource, interpreter, token) + const env = + this.interpreterService.known.find((e) => e.id === interpreter.id) || + (await this.interpreterService.resolveEnvironment(interpreter.id)); + if (!env) { + return; + } + const title = DataScience.activatingPythonEnvironment(getPythonEnvDisplayName(env)); + return KernelProgressReporter.wrapAndReportProgress(resource, title, async () => + this.getActivatedEnvironmentVariablesImplWithCaching( + resource, + (await this.interpreterService.resolveEnvironment(env)) || env, + token + ) ); } private async getActivatedEnvironmentVariablesImplWithCaching( resource: Resource, - interpreter: PythonEnvironment, + environment: Environment, token?: CancellationToken ): Promise { - const key = `${resource?.toString() || ''}${interpreter.id}`; + const key = `${resource?.toString() || ''}${environment.id}`; const info = this.activatedEnvVariablesCache.get(key); if (info && info.time.elapsedTime >= ENV_VAR_CACHE_TIMEOUT) { this.activatedEnvVariablesCache.delete(key); } if (!this.activatedEnvVariablesCache.has(key)) { - const promise = this.getActivatedEnvironmentVariablesImpl(resource, interpreter, token); + const promise = this.getActivatedEnvironmentVariablesImpl(resource, environment, token); promise.catch(noop); this.activatedEnvVariablesCache.set(key, { promise, time: new StopWatch() }); } @@ -90,14 +100,14 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi @traceDecoratorVerbose('Getting activated env variables', TraceOptions.BeforeCall | TraceOptions.Arguments) private async getActivatedEnvironmentVariablesImpl( resource: Resource, - @logValue('uri') interpreter: PythonEnvironment, + @logValue('id') environment: Environment, token?: CancellationToken ): Promise { if (!this.extensionChecker.isPythonExtensionInstalled) { return; } const stopWatch = new StopWatch(); - return this.getActivatedEnvironmentVariablesFromPython(resource, interpreter, token) + return this.getActivatedEnvironmentVariablesFromPython(resource, environment, token) .then((env) => { if (token?.isCancellationRequested) { return; @@ -106,7 +116,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi }) .catch((ex) => { traceError( - `Failed to get env vars with python ${getDisplayPath(interpreter?.uri)} in ${ + `Failed to get env vars with python ${getDisplayPath(environment.id)} in ${ stopWatch.elapsedTime }ms`, ex @@ -121,10 +131,10 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi >(); private async getActivatedEnvironmentVariablesFromPython( resource: Resource, - @logValue('uri') interpreter: PythonEnvironment, + @logValue<{ id: string }>('id') environment: Environment, @ignoreLogging() token?: CancellationToken ): Promise { - const key = `${resource?.toString() || ''}${interpreter?.id || ''}`; + const key = `${resource?.toString() || ''}${environment.id || ''}`; // Ensure the cache is only valid for a limited time. const info = this.cachedEnvVariables.get(key); @@ -132,7 +142,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi this.cachedEnvVariables.delete(key); } if (!this.cachedEnvVariables.has(key)) { - const promise = this.getActivatedEnvironmentVariablesFromPythonImpl(resource, interpreter, token); + const promise = this.getActivatedEnvironmentVariablesFromPythonImpl(resource, environment, token); this.cachedEnvVariables.set(key, { promise, lastRequestedTime: new StopWatch() }); } @@ -140,7 +150,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi } private async getActivatedEnvironmentVariablesFromPythonImpl( resource: Resource, - interpreter: PythonEnvironment, + environment: Environment, token?: CancellationToken ): Promise { resource = resource @@ -163,11 +173,15 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi let env = await this.apiProvider.getApi().then((api) => api - .getActivatedEnvironmentVariables(resource, serializePythonEnvironment(interpreter)!, false) + .getActivatedEnvironmentVariables( + resource, + serializePythonEnvironment(pythonEnvToJupyterEnv(environment, true))!, + false + ) .catch((ex) => { traceError( `Failed to get activated env variables from Python Extension for ${getDisplayPath( - interpreter.uri + environment.path )}`, ex ); @@ -178,7 +192,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi if (token?.isCancellationRequested) { return; } - const envType = interpreter.envType; + const envType = getEnvironmentType(environment); sendTelemetryEvent( Telemetry.GetActivatedEnvironmentVariables, { duration: stopWatch.elapsedTime }, @@ -193,7 +207,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi if (env) { traceVerbose( - `Got env vars with python ${getDisplayPath(interpreter?.uri)}, with env var count ${ + `Got env vars with python ${getDisplayPath(environment.path)}, with env var count ${ Object.keys(env || {}).length } in ${stopWatch.elapsedTime}ms. \n PATH value is ${env.PATH} and \n Path value is ${env.Path}` ); @@ -201,23 +215,32 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi // We must get activated env variables for Conda env, if not running stuff against conda will not work. // Hence we must log these as errors (so we can see them in jupyter logs). traceError( - `Failed to get activated conda env vars for ${getDisplayPath(interpreter?.uri)} + `Failed to get activated conda env vars for ${getDisplayPath(environment.path)} in ${stopWatch.elapsedTime}ms` ); } else { traceWarning( - `Failed to get activated env vars for ${getDisplayPath(interpreter?.uri)} in ${stopWatch.elapsedTime}ms` + `Failed to get activated env vars for ${getDisplayPath(environment.path)} in ${stopWatch.elapsedTime}ms` ); } if (!env) { - // Temporary work around until https://github.com/microsoft/vscode-python/issues/20663 + // Temporary work around until https://github.com/microsoft/vscode-python/issues/20678 // However we might still need a work around for failure to activate conda envs without Python. const customEnvVars = await customEnvVarsPromise; env = {}; // Patch for conda envs. - if (interpreter.envType === EnvironmentType.Conda && interpreter.sysPrefix) { - env.CONDA_PREFIX = interpreter.sysPrefix; + if (getEnvironmentType(environment) === EnvironmentType.Conda) { + const sysPrefix = + this.interpreterService.known.find((e) => e.id === environment.id)?.executable.sysPrefix || + (await this.interpreterService.resolveEnvironment(environment))?.executable.sysPrefix; + if (sysPrefix) { + env.CONDA_PREFIX = sysPrefix; + } else { + traceWarning( + `Failed to get the SysPrefix for the Conda Environment ${getDisplayPath(environment.path)}}` + ); + } } this.envVarsService.mergeVariables(process.env, env); // Copy current proc vars into new obj. @@ -234,7 +257,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi this.envVarsService.appendPythonPath(env, customEnvVars!.PYTHONPATH); } - const executablesPath = await this.globalExecPaths.getExecutablesPath(interpreter).catch(noop); + const executablesPath = await this.globalExecPaths.getExecutablesPath(environment).catch(noop); if (token?.isCancellationRequested) { return; } @@ -245,19 +268,19 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi // Second value in PATH is expected to be the site packages directory. if (executablesPath && pathValues[1] !== executablesPath.fsPath) { traceVerbose( - `Prepend PATH with user site path for ${getDisplayPath(interpreter.id)}, user site ${ + `Prepend PATH with user site path for ${getDisplayPath(environment.path)}, user site ${ executablesPath.fsPath }` ); // Based on docs this is the right path and must be setup in the path. this.envVarsService.prependPath(env, executablesPath.fsPath); - } else if (interpreter.isCondaEnvWithoutPython) { + } else if (isCondaEnvironmentWithoutPython(environment)) { // } else { traceError( - `Unable to determine site packages path for python ${getDisplayPath(interpreter.uri.fsPath)} (${ - interpreter.envType - })` + `Unable to determine site packages path for python ${getDisplayPath( + environment.path + )} (${getEnvironmentType(environment)})` ); } @@ -272,7 +295,7 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi // Ensure the first path in PATH variable points to the directory of python executable. // We need to add this to ensure kernels start and work correctly, else things can fail miserably. traceVerbose( - `Prepend PATH with python bin for ${getDisplayPath(interpreter.id)}, \n PATH value is ${ + `Prepend PATH with python bin for ${getDisplayPath(environment.path)}, \n PATH value is ${ env.PATH } and \n Path value is ${env.Path}` ); @@ -280,10 +303,12 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi // This way shell commands such as `!pip`, `!python` end up pointing to the right executables. // Also applies to `!java` where java could be an executable in the conda bin directory. // Also required for conda environments that do not have Python installed (in the conda env). - this.envVarsService.prependPath(env, path.dirname(interpreter.uri.fsPath)); + if (environment.executable.uri) { + this.envVarsService.prependPath(env, path.dirname(environment.executable.uri.fsPath)); + } traceVerbose( - `Activated Env Variables for ${getDisplayPath(interpreter.id)}, \n PATH value is ${ + `Activated Env Variables for ${getDisplayPath(environment.path)}, \n PATH value is ${ env.PATH } and \n Path value is ${env.Path}` ); diff --git a/src/platform/interpreter/globalPythonExePathService.node.ts b/src/platform/interpreter/globalPythonExePathService.node.ts index 518d4af89a9..60800023f4b 100644 --- a/src/platform/interpreter/globalPythonExePathService.node.ts +++ b/src/platform/interpreter/globalPythonExePathService.node.ts @@ -4,13 +4,15 @@ import { inject, injectable } from 'inversify'; import { Uri } from 'vscode'; import * as path from '../vscode-path/resources'; -import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info'; +import { EnvironmentType } from '../pythonEnvironments/info'; import { IFileSystem, IPlatformService } from '../common/platform/types'; import { ResourceMap } from '../common/resourceMap'; import { swallowExceptions } from '../common/utils/decorators'; import { IProcessServiceFactory } from '../common/process/types.node'; import { traceVerbose } from '../logging'; import { getDisplayPath } from '../common/platform/fs-paths'; +import { Environment } from '../api/pythonApiTypes'; +import { getEnvironmentType } from './helpers'; @injectable() export class GlobalPythonExecutablePathService { @@ -25,21 +27,22 @@ export class GlobalPythonExecutablePathService { * Gets the path where executables are installed for the given Global Python interpreter. */ @swallowExceptions() - public async getExecutablesPath(interpreter: PythonEnvironment): Promise { - if (interpreter.envType !== EnvironmentType.Unknown) { + public async getExecutablesPath(environment: Environment): Promise { + const executable = environment.executable.uri; + if (getEnvironmentType(environment) !== EnvironmentType.Unknown || !executable) { return; } - if (!this.userSitePaths.has(interpreter.uri)) { - const promise = this.getUserSitePathImpl(interpreter); + if (!this.userSitePaths.has(executable)) { + const promise = this.getUserSitePathImpl(executable); promise.catch(() => { - if (this.userSitePaths.get(interpreter.uri) === promise) { - this.userSitePaths.delete(interpreter.uri); + if (this.userSitePaths.get(executable) === promise) { + this.userSitePaths.delete(executable); } }); - this.userSitePaths.set(interpreter.uri, promise); + this.userSitePaths.set(executable, promise); } - return this.userSitePaths.get(interpreter.uri); + return this.userSitePaths.get(executable); } /** * Tested the following scenarios: @@ -65,12 +68,12 @@ export class GlobalPythonExecutablePathService { * On Unix it is USER_BASE/bin * */ - private async getUserSitePathImpl(interpreter: PythonEnvironment): Promise { + private async getUserSitePathImpl(executable: Uri): Promise { const processService = await this.processFactory.create(undefined); const delimiter = 'USER_BASE_VALUE'; const valueToUse = this.platform.isWindows ? 'USER_SITE' : 'USER_BASE'; // Add delimiters as sometimes, the python runtime can spit out warning/information messages as well. - const { stdout } = await processService.exec(interpreter.uri.fsPath, [ + const { stdout } = await processService.exec(executable.fsPath, [ '-c', `import site;print("${delimiter}");print(site.${valueToUse});print("${delimiter}");` ]); @@ -87,17 +90,13 @@ export class GlobalPythonExecutablePathService { } if (!sitePath || !this.fs.exists(sitePath)) { throw new Error( - `USER_SITE ${sitePath.fsPath} dir does not exist for the interpreter ${getDisplayPath( - interpreter.uri - )}` + `USER_SITE ${sitePath.fsPath} dir does not exist for the interpreter ${getDisplayPath(executable)}` ); } - traceVerbose(`USER_SITE for ${getDisplayPath(interpreter.uri)} is ${sitePath.fsPath}`); + traceVerbose(`USER_SITE for ${getDisplayPath(executable)} is ${sitePath.fsPath}`); return sitePath; } else { - throw new Error( - `USER_SITE not found for the interpreter ${getDisplayPath(interpreter.uri)}. Stdout: ${stdout}` - ); + throw new Error(`USER_SITE not found for the interpreter ${getDisplayPath(executable)}. Stdout: ${stdout}`); } } } diff --git a/src/platform/interpreter/helpers.ts b/src/platform/interpreter/helpers.ts index 61b2aac2f6f..570deeebc8a 100644 --- a/src/platform/interpreter/helpers.ts +++ b/src/platform/interpreter/helpers.ts @@ -4,7 +4,7 @@ import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info'; import { getTelemetrySafeVersion } from '../telemetry/helpers'; import { basename } from '../../platform/vscode-path/resources'; -import { Environment } from '../api/pythonApiTypes'; +import { Environment, KnownEnvironmentTools, KnownEnvironmentTypes } from '../api/pythonApiTypes'; export function getPythonEnvDisplayName(interpreter: PythonEnvironment | Environment) { if ('executable' in interpreter) { @@ -68,7 +68,28 @@ const environmentTypes = [ EnvironmentType.VirtualEnv, EnvironmentType.VirtualEnvWrapper ]; + export function getEnvironmentType(env: Environment): EnvironmentType { + if ((env.environment?.type as KnownEnvironmentTypes) === 'Conda') { + return EnvironmentType.Conda; + } + + // Map the Python env tool to a Jupyter environment type. + const orderOrEnvs: [pythonEnvTool: KnownEnvironmentTools, JupyterEnv: EnvironmentType][] = [ + ['Conda', EnvironmentType.Conda], + ['Pyenv', EnvironmentType.Pyenv], + ['Pipenv', EnvironmentType.Pipenv], + ['Poetry', EnvironmentType.Poetry], + ['VirtualEnvWrapper', EnvironmentType.VirtualEnvWrapper], + ['VirtualEnv', EnvironmentType.VirtualEnv], + ['Venv', EnvironmentType.Venv] + ]; + for (const [pythonEnvTool, JupyterEnv] of orderOrEnvs) { + if (env.tools.includes(pythonEnvTool)) { + return JupyterEnv; + } + } + for (const type of environmentTypes) { if (env.tools.some((tool) => tool.toLowerCase() === type.toLowerCase())) { return type; From a19acdbb5175db38a4b25d24f9c15184585e9925 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 4 Aug 2023 12:09:34 +1000 Subject: [PATCH 2/2] Support new Python API types in installers --- src/kernels/errors/kernelErrorHandler.ts | 7 +++- src/platform/common/utils/misc.ts | 3 +- ...InstalledWindowsLongPathNotEnabledError.ts | 3 +- .../installer/condaInstaller.node.ts | 39 ++++++++++++++----- src/platform/interpreter/installer/helpers.ts | 7 +++- .../installer/moduleInstaller.node.ts | 7 ++-- .../installer/pipEnvInstaller.node.ts | 10 +++-- .../installer/pipInstaller.node.ts | 16 +++++--- .../installer/poetryInstaller.node.ts | 18 +++++++-- .../installer/productInstaller.node.ts | 8 +++- .../installer/productInstaller.unit.test.ts | 3 +- src/platform/interpreter/installer/types.ts | 9 +++-- src/platform/interpreter/types.node.ts | 5 ++- 13 files changed, 96 insertions(+), 39 deletions(-) diff --git a/src/kernels/errors/kernelErrorHandler.ts b/src/kernels/errors/kernelErrorHandler.ts index 61ab346b4e7..81677106960 100644 --- a/src/kernels/errors/kernelErrorHandler.ts +++ b/src/kernels/errors/kernelErrorHandler.ts @@ -68,6 +68,7 @@ import { PackageNotInstalledWindowsLongPathNotEnabledError } from '../../platfor import { JupyterNotebookNotInstalled } from '../../platform/errors/jupyterNotebookNotInstalled'; import { fileToCommandArgument } from '../../platform/common/helpers'; import { getJupyterDisplayName } from '../jupyter/connection/jupyterUriProviderRegistration'; +import { getPythonEnvDisplayName } from '../../platform/interpreter/helpers'; /*** * Common code for handling errors. @@ -160,8 +161,10 @@ export abstract class DataScienceErrorHandler implements IDataScienceErrorHandle typeof error.product === 'string' ? error.product : ProductNames.get(error.product) || `${error.product}`; - const interpreterDisplayName = error.interpreter.displayName || error.interpreter.envName || ''; - const displayPath = getDisplayPath(error.interpreter.uri); + const interpreterDisplayName = getPythonEnvDisplayName(error.interpreter) || error.interpreter.id || ''; + const displayPath = getDisplayPath( + 'executable' in error.interpreter ? error.interpreter.executable.uri : error.interpreter.uri + ); let displayName = interpreterDisplayName ? ` ${interpreterDisplayName} (${displayPath})` : displayPath; return DataScience.packageNotInstalledWindowsLongPathNotEnabledError(packageName, displayName); } else if ( diff --git a/src/platform/common/utils/misc.ts b/src/platform/common/utils/misc.ts index 7d59466d162..7c26f068683 100644 --- a/src/platform/common/utils/misc.ts +++ b/src/platform/common/utils/misc.ts @@ -6,6 +6,7 @@ import { NotebookCellScheme } from '../constants'; import { InterpreterUri, Resource } from '../types'; import { isPromise } from './async'; import { StopWatch } from './stopWatch'; +import { Environment } from '../../api/pythonApiTypes'; // eslint-disable-next-line no-empty,@typescript-eslint/no-empty-function export function noop() {} @@ -105,7 +106,7 @@ export function tracing(log: (t: TraceInfo) => void, run: () => T, logBeforeC * @param {InterpreterUri} [resource] * @returns {resource is Resource} */ -export function isResource(resource?: InterpreterUri): resource is Resource { +export function isResource(resource?: InterpreterUri | Environment): resource is Resource { if (!resource) { return true; } diff --git a/src/platform/errors/packageNotInstalledWindowsLongPathNotEnabledError.ts b/src/platform/errors/packageNotInstalledWindowsLongPathNotEnabledError.ts index 142924d0ccf..4e3c61b93d1 100644 --- a/src/platform/errors/packageNotInstalledWindowsLongPathNotEnabledError.ts +++ b/src/platform/errors/packageNotInstalledWindowsLongPathNotEnabledError.ts @@ -4,13 +4,14 @@ import { BaseError } from './types'; import { PythonEnvironment } from '../pythonEnvironments/info'; import { Product } from '../interpreter/installer/types'; +import { Environment } from '../api/pythonApiTypes'; /** * Thrown when we fail to install a Package due to long path not being enabled on Windows. */ export class PackageNotInstalledWindowsLongPathNotEnabledError extends BaseError { constructor( public readonly product: Product | string, - public readonly interpreter: PythonEnvironment, + public readonly interpreter: PythonEnvironment | Environment, public readonly originalMessage: string ) { super('windowsLongPathNotEnabled', originalMessage); diff --git a/src/platform/interpreter/installer/condaInstaller.node.ts b/src/platform/interpreter/installer/condaInstaller.node.ts index 3025b6eeaa0..b36f71e6b05 100644 --- a/src/platform/interpreter/installer/condaInstaller.node.ts +++ b/src/platform/interpreter/installer/condaInstaller.node.ts @@ -11,9 +11,11 @@ import * as path from '../../vscode-path/path'; import { translateProductToModule } from './utils'; import { fileToCommandArgument, toCommandArgument } from '../../common/helpers'; import { getPinnedPackages } from './pinnedPackages'; -import { CancellationTokenSource } from 'vscode'; +import { CancellationTokenSource, Uri } from 'vscode'; import { IPythonExtensionChecker } from '../../api/types'; import { IInterpreterService } from '../contracts'; +import { Environment } from '../../api/pythonApiTypes'; +import { getEnvironmentType, isCondaEnvironmentWithoutPython } from '../helpers'; /** * A Python module installer for a conda environment. @@ -53,7 +55,7 @@ export class CondaInstaller extends ModuleInstaller { * @param {InterpreterUri} [resource=] Resource used to identify the workspace. * @returns {Promise} Whether conda is supported as a module installer or not. */ - public async isSupported(interpreter: PythonEnvironment): Promise { + public async isSupported(interpreter: PythonEnvironment | Environment): Promise { if (this._isCondaAvailable === false) { return false; } @@ -63,12 +65,15 @@ export class CondaInstaller extends ModuleInstaller { return false; } // Now we need to check if the current environment is a conda environment or not. - return interpreter.envType === EnvironmentType.Conda; + return ( + ('executable' in interpreter ? getEnvironmentType(interpreter) : interpreter.envType) === + EnvironmentType.Conda + ); } public override async installModule( productOrModuleName: Product | string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, cancelTokenSource: CancellationTokenSource, flags?: ModuleInstallFlags ): Promise { @@ -76,7 +81,14 @@ export class CondaInstaller extends ModuleInstaller { // If we just installed a package into a conda env without python init, then Python may have gotten installed // We now need to ensure the conda env gets updated as a result of this. - if (interpreter.envType === EnvironmentType.Conda && interpreter.isCondaEnvWithoutPython) { + if ( + ('executable' in interpreter + ? getEnvironmentType(interpreter) + : interpreter.envType === EnvironmentType.Conda) && + ('executable' in interpreter + ? isCondaEnvironmentWithoutPython(interpreter) + : interpreter.isCondaEnvWithoutPython) + ) { const pythonExt = this.serviceContainer.get(IPythonExtensionChecker); if (!pythonExt.isPythonExtensionActive) { return; @@ -94,12 +106,12 @@ export class CondaInstaller extends ModuleInstaller { */ protected async getExecutionArgs( moduleName: string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, flags: ModuleInstallFlags = ModuleInstallFlags.None ): Promise { const condaService = this.serviceContainer.get(CondaService); const condaFile = await condaService.getCondaFile(); - const name = interpreter.envName; + const name = 'executable' in interpreter ? interpreter.environment?.name : interpreter.envName; const envPath = this.getEnvironmentPath(interpreter); const args = [flags & ModuleInstallFlags.upgrade ? 'update' : 'install']; @@ -137,8 +149,17 @@ export class CondaInstaller extends ModuleInstaller { }; } - private getEnvironmentPath(interpreter: PythonEnvironment) { - const dir = path.dirname(interpreter.uri.fsPath); + private getEnvironmentPath(interpreter: PythonEnvironment | Environment) { + let exeuctablePath: Uri; + if ('executable' in interpreter) { + if (interpreter.environment?.folderUri) { + return interpreter.environment.folderUri.fsPath; + } + exeuctablePath = interpreter.executable.uri || Uri.file(interpreter.path); + } else { + exeuctablePath = interpreter.uri; + } + const dir = path.dirname(exeuctablePath.fsPath); // If interpreter is in bin or Scripts, then go up one level const subDirName = path.basename(dir); diff --git a/src/platform/interpreter/installer/helpers.ts b/src/platform/interpreter/installer/helpers.ts index cd61d3e59fa..26e0113bc6e 100644 --- a/src/platform/interpreter/installer/helpers.ts +++ b/src/platform/interpreter/installer/helpers.ts @@ -4,14 +4,17 @@ import { Uri } from 'vscode'; import { IWorkspaceService } from '../../common/application/types'; import { PythonEnvironment } from '../../pythonEnvironments/info'; +import { Environment } from '../../api/pythonApiTypes'; /** * Returns the workspace folder this interpreter is based in or the root if not a virtual env */ export function getInterpreterWorkspaceFolder( - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, workspaceService: IWorkspaceService ): Uri | undefined { - const folder = workspaceService.getWorkspaceFolder(interpreter.uri); + const uri = + 'executable' in interpreter ? interpreter.executable.uri || Uri.file(interpreter.path) : interpreter.uri; + const folder = workspaceService.getWorkspaceFolder(uri); return folder?.uri || workspaceService.rootFolder; } diff --git a/src/platform/interpreter/installer/moduleInstaller.node.ts b/src/platform/interpreter/installer/moduleInstaller.node.ts index 35d92608db8..fb6dc8d5f60 100644 --- a/src/platform/interpreter/installer/moduleInstaller.node.ts +++ b/src/platform/interpreter/installer/moduleInstaller.node.ts @@ -17,6 +17,7 @@ import { EOL } from 'os'; import { PackageNotInstalledWindowsLongPathNotEnabledError } from '../../errors/packageNotInstalledWindowsLongPathNotEnabledError'; import { splitLines } from '../../common/helpers'; import { IPythonExecutionFactory } from '../types.node'; +import { Environment } from '../../api/pythonApiTypes'; export type ExecutionInstallArgs = { args: string[]; @@ -38,7 +39,7 @@ export abstract class ModuleInstaller implements IModuleInstaller { public async installModule( productOrModuleName: Product | string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, cancelTokenSource: CancellationTokenSource, flags?: ModuleInstallFlags ): Promise { @@ -193,10 +194,10 @@ export abstract class ModuleInstaller implements IModuleInstaller { }; await shell.withProgress(options, async (progress, token: CancellationToken) => install(progress, token)); } - public abstract isSupported(interpreter: PythonEnvironment): Promise; + public abstract isSupported(interpreter: PythonEnvironment | Environment): Promise; protected abstract getExecutionArgs( moduleName: string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, flags?: ModuleInstallFlags ): Promise; } diff --git a/src/platform/interpreter/installer/pipEnvInstaller.node.ts b/src/platform/interpreter/installer/pipEnvInstaller.node.ts index 24d029db641..2098d66714a 100644 --- a/src/platform/interpreter/installer/pipEnvInstaller.node.ts +++ b/src/platform/interpreter/installer/pipEnvInstaller.node.ts @@ -13,6 +13,8 @@ import { isPipenvEnvironmentRelatedToFolder } from './pipenv.node'; import { IServiceContainer } from '../../ioc/types'; import { getFilePath } from '../../common/platform/fs-paths'; import { getInterpreterWorkspaceFolder } from './helpers'; +import { Environment } from '../../api/pythonApiTypes'; +import { getEnvironmentType } from '../helpers'; export const pipenvName = 'pipenv'; @@ -42,7 +44,7 @@ export class PipEnvInstaller extends ModuleInstaller { return 10; } - public async isSupported(resource?: InterpreterUri): Promise { + public async isSupported(resource?: InterpreterUri | Environment): Promise { if (isResource(resource)) { const interpreter = await this.serviceContainer .get(IInterpreterService) @@ -56,12 +58,14 @@ export class PipEnvInstaller extends ModuleInstaller { // Install using `pipenv install` only if the active environment is related to the current folder. return isPipenvEnvironmentRelatedToFolder(interpreter.uri, workspaceFolder.uri); } else { - return resource.envType === EnvironmentType.Pipenv; + return ( + ('executable' in resource ? getEnvironmentType(resource) : resource.envType) === EnvironmentType.Pipenv + ); } } protected async getExecutionArgs( moduleName: string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, flags: ModuleInstallFlags = 0 ): Promise { // In pipenv the only way to update/upgrade or re-install is update (apart from a complete uninstall and re-install). diff --git a/src/platform/interpreter/installer/pipInstaller.node.ts b/src/platform/interpreter/installer/pipInstaller.node.ts index a4e46c882ce..93165b37d1d 100644 --- a/src/platform/interpreter/installer/pipInstaller.node.ts +++ b/src/platform/interpreter/installer/pipInstaller.node.ts @@ -12,6 +12,8 @@ import { IServiceContainer } from '../../ioc/types'; import { translateProductToModule } from './utils'; import { IPythonExecutionFactory } from '../types.node'; import { getPinnedPackages } from './pinnedPackages'; +import { Environment } from '../../api/pythonApiTypes'; +import { getEnvironmentType } from '../helpers'; /** * Installer for pip. Default installer for most everything. @@ -37,9 +39,10 @@ export class PipInstaller extends ModuleInstaller { public get priority(): number { return 0; } - public async isSupported(interpreter: PythonEnvironment): Promise { + public async isSupported(interpreter: PythonEnvironment | Environment): Promise { + const envType = 'executable' in interpreter ? getEnvironmentType(interpreter) : interpreter.envType; // Skip this on conda, poetry, and pipenv environments - switch (interpreter.envType) { + switch (envType) { case EnvironmentType.Conda: case EnvironmentType.Pipenv: case EnvironmentType.Poetry: @@ -51,7 +54,7 @@ export class PipInstaller extends ModuleInstaller { } protected async getExecutionArgs( moduleName: string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, flags: ModuleInstallFlags = 0 ): Promise { if (moduleName === translateProductToModule(Product.pip)) { @@ -80,14 +83,17 @@ export class PipInstaller extends ModuleInstaller { if (flags & ModuleInstallFlags.reInstall) { args.push('--force-reinstall'); } - if (interpreter.envType === EnvironmentType.Unknown) { + if ( + ('executable' in interpreter ? getEnvironmentType(interpreter) : interpreter.envType) === + EnvironmentType.Unknown + ) { args.push('--user'); } return { args: ['-m', 'pip', ...args, moduleName].concat(getPinnedPackages('pip', moduleName)) }; } - private isPipAvailable(interpreter: PythonEnvironment): Promise { + private isPipAvailable(interpreter: PythonEnvironment | Environment): Promise { const pythonExecutionFactory = this.serviceContainer.get(IPythonExecutionFactory); return pythonExecutionFactory .create({ resource: undefined, interpreter }) diff --git a/src/platform/interpreter/installer/poetryInstaller.node.ts b/src/platform/interpreter/installer/poetryInstaller.node.ts index bcdc1e2df41..e3a5b82fab9 100644 --- a/src/platform/interpreter/installer/poetryInstaller.node.ts +++ b/src/platform/interpreter/installer/poetryInstaller.node.ts @@ -10,6 +10,9 @@ import { IServiceContainer } from '../../ioc/types'; import { ExecutionInstallArgs, ModuleInstaller } from './moduleInstaller.node'; import { isPoetryEnvironmentRelatedToFolder } from './poetry.node'; import { ModuleInstallerType } from './types'; +import { Environment } from '../../api/pythonApiTypes'; +import { Uri } from 'vscode'; +import { getEnvironmentType } from '../helpers'; export const poetryName = 'poetry'; @@ -46,16 +49,23 @@ export class PoetryInstaller extends ModuleInstaller { super(serviceContainer); } - public async isSupported(interpreter: PythonEnvironment): Promise { - if (interpreter.envType !== EnvironmentType.Poetry) { + public async isSupported(interpreter: PythonEnvironment | Environment): Promise { + if ( + ('executable' in interpreter ? getEnvironmentType(interpreter) : interpreter.envType) !== + EnvironmentType.Poetry + ) { return false; } const folder = getInterpreterWorkspaceFolder(interpreter, this.workspaceService); if (folder) { + const executable = + 'executable' in interpreter + ? interpreter.executable.uri || Uri.file(interpreter.path) + : interpreter.uri; // Install using poetry CLI only if the active poetry environment is related to the current folder. return isPoetryEnvironmentRelatedToFolder( - interpreter.uri.fsPath, + executable.fsPath, folder.fsPath, this.configurationService.getSettings(undefined).poetryPath ); @@ -66,7 +76,7 @@ export class PoetryInstaller extends ModuleInstaller { protected async getExecutionArgs( moduleName: string, - interpreter: PythonEnvironment + interpreter: PythonEnvironment | Environment ): Promise { const execPath = this.configurationService.getSettings(undefined).poetryPath; const args = [execPath, 'add', '--dev', moduleName]; diff --git a/src/platform/interpreter/installer/productInstaller.node.ts b/src/platform/interpreter/installer/productInstaller.node.ts index 99b92989c8b..4529e03ae02 100644 --- a/src/platform/interpreter/installer/productInstaller.node.ts +++ b/src/platform/interpreter/installer/productInstaller.node.ts @@ -38,6 +38,7 @@ import { trackPackageInstalledIntoInterpreter } from './productInstaller'; import { translateProductToModule } from './utils'; import { IInterpreterPackages } from '../types'; import { IPythonExecutionFactory } from '../types.node'; +import { Environment } from '../../api/pythonApiTypes'; export async function isModulePresentInEnvironment(memento: Memento, product: Product, interpreter: PythonEnvironment) { const key = `${await getInterpreterHash(interpreter)}#${ProductNames.get(product)}`; @@ -125,7 +126,10 @@ export class DataScienceInstaller { } @traceDecoratorVerbose('Checking if product is installed') - public async isInstalled(product: Product, @logValue('path') interpreter: PythonEnvironment): Promise { + public async isInstalled( + product: Product, + @logValue('path') interpreter: PythonEnvironment | Environment + ): Promise { const executableName = this.getExecutableNameFromSettings(product, undefined); const isModule = this.isExecutableAModule(product, undefined); if (isModule) { @@ -235,7 +239,7 @@ export class ProductInstaller implements IInstaller { } } - public async isInstalled(product: Product, interpreter: PythonEnvironment): Promise { + public async isInstalled(product: Product, interpreter: PythonEnvironment | Environment): Promise { return this.createInstaller(product).isInstalled(product, interpreter); } diff --git a/src/platform/interpreter/installer/productInstaller.unit.test.ts b/src/platform/interpreter/installer/productInstaller.unit.test.ts index 384d92b70e4..452516fddca 100644 --- a/src/platform/interpreter/installer/productInstaller.unit.test.ts +++ b/src/platform/interpreter/installer/productInstaller.unit.test.ts @@ -19,10 +19,11 @@ import { ModuleInstallerType } from '../../../platform/interpreter/installer/types'; import { sleep } from '../../../test/core'; +import { Environment } from '../../api/pythonApiTypes'; class AlwaysInstalledDataScienceInstaller extends DataScienceInstaller { // eslint-disable-next-line @typescript-eslint/no-unused-vars, class-methods-use-this - public override async isInstalled(_product: Product, _resource?: InterpreterUri): Promise { + public override async isInstalled(_product: Product, _resource?: InterpreterUri | Environment): Promise { return true; } } diff --git a/src/platform/interpreter/installer/types.ts b/src/platform/interpreter/installer/types.ts index 830a61e580d..dbfbfd18607 100644 --- a/src/platform/interpreter/installer/types.ts +++ b/src/platform/interpreter/installer/types.ts @@ -4,6 +4,7 @@ import { CancellationTokenSource, Event, Uri } from 'vscode'; import { InterpreterUri } from '../../common/types'; import { PythonEnvironment } from '../../pythonEnvironments/info'; +import { Environment } from '../../api/pythonApiTypes'; export enum InstallerResponse { Installed, @@ -72,7 +73,7 @@ export interface IModuleInstaller { */ installModule( product: string, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, cancelTokenSource: CancellationTokenSource, flags?: ModuleInstallFlags ): Promise; @@ -89,11 +90,11 @@ export interface IModuleInstaller { */ installModule( product: Product, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | Environment, cancelTokenSource: CancellationTokenSource, flags?: ModuleInstallFlags ): Promise; - isSupported(resource?: InterpreterUri): Promise; + isSupported(resource?: InterpreterUri | Environment): Promise; } export const IPythonInstallation = Symbol('IPythonInstallation'); @@ -136,6 +137,6 @@ export interface IInstaller { reInstallAndUpdate?: boolean, installPipIfRequired?: boolean ): Promise; - isInstalled(product: Product, resource: InterpreterUri): Promise; + isInstalled(product: Product, resource: InterpreterUri | Environment): Promise; translateProductToModuleName(product: Product, purpose: ModuleNamePurpose): string; } diff --git a/src/platform/interpreter/types.node.ts b/src/platform/interpreter/types.node.ts index 9a5a7f17b8f..69d35be9211 100644 --- a/src/platform/interpreter/types.node.ts +++ b/src/platform/interpreter/types.node.ts @@ -4,15 +4,16 @@ import { Uri } from 'vscode'; import { ExecutionResult, ObservableExecutionResult, SpawnOptions } from '../common/process/types.node'; import { PythonEnvironment } from '../pythonEnvironments/info'; +import { Environment } from '../api/pythonApiTypes'; export type ExecutionFactoryCreationOptions = { resource?: Uri; - interpreter: PythonEnvironment; + interpreter: PythonEnvironment | Environment; }; export type ExecutionFactoryCreateWithEnvironmentOptions = { resource?: Uri; - interpreter: PythonEnvironment; + interpreter: PythonEnvironment | Environment; allowEnvironmentFetchExceptions?: boolean; }; export const IPythonExecutionFactory = Symbol('IPythonExecutionFactory');