Fix issues loading stdlib packages on newer Python versions.

src/pyodide/internal/loadPackage.ts

@@ -140,11 +140,15 @@ async function loadPackagesImpl(
   Module.FS.mount(tarFS, { info }, path);
 }
 
+/**
+ * Downloads the requirements specified and loads them into Pyodide. Note that this does not
+ * do any dependency resolution, it just installs the requirements that are specified. See
+ * `getTransitiveRequirements` for the code that deals with this.
+ */
 export async function loadPackages(Module: Module, requirements: Set<string>) {
-  const pkgsToLoad = requirements.union(new Set(STDLIB_PACKAGES));
   if (LOAD_WHEELS_FROM_R2) {
-    await loadPackagesImpl(Module, pkgsToLoad, loadBundleFromR2);
+    await loadPackagesImpl(Module, requirements, loadBundleFromR2);
   } else if (LOAD_WHEELS_FROM_ARTIFACT_BUNDLER) {
-    await loadPackagesImpl(Module, pkgsToLoad, loadBundleFromArtifactBundler);
+    await loadPackagesImpl(Module, requirements, loadBundleFromArtifactBundler);
   }
 }

src/pyodide/internal/setupPackages.ts

@@ -24,7 +24,7 @@ function canonicalizePackageName(name: string): string {
 
 // The "name" field in the lockfile is not canonicalized
 export const STDLIB_PACKAGES: string[] = Object.values(LOCKFILE.packages)
-  .filter(({ install_dir }) => install_dir === 'stdlib')
+  .filter(({ package_type }) => package_type === 'cpython_module')
   .map(({ name }) => canonicalizePackageName(name));
 
 // Each item in the list is an element of the file path, for example
@@ -183,9 +183,20 @@ function disabledLoadPackage(): never {
  * Get the set of transitive requirements from the REQUIREMENTS metadata.
  */
 function getTransitiveRequirements(): Set<string> {
-  const requirements = REQUIREMENTS.map(canonicalizePackageName);
+  // We have had a regression where the lockfile format changed and so no stdlib packages were
+  // found. This created a strange and unfriendly error message, so we throw a nice error here if
+  // there are no stdlib packages to avoid this if it happens in the future.
+  if (STDLIB_PACKAGES.length == 0) {
+    throw new Error(
+      'Found no STDLIB_PACKAGES, expected at least 1. Lock file corrupted?'
+    );
+  }
+  const requirements = Array.from(
+    new Set(REQUIREMENTS).union(new Set(STDLIB_PACKAGES)),
+    canonicalizePackageName
+  );
+
   // resolve transitive dependencies of requirements and if IN_WORKERD install them from the cdn.
-  // TODO(later): use current package's LOCKFILE instead of the global.
   const packageDatas = recursiveDependencies(LOCKFILE, requirements);
   return new Set(packageDatas.map(({ name }) => canonicalizePackageName(name)));
 }