Exclude ipykernel native libs for macOS universal release (#6377)

Fixing macOS releases. 

Skipping comparing files from the bundled ipykernel while preparing
macOS universal release.

To expedite fixing desktop macOS builds, it was decided to not try to
bundle ipykernel for both arm64 AND x86 macOS. Wasim updated the PR so
that we only bundle ipykernel for arm64 macOS for now. Intel x86
environments will need to continue install ipykernel on first use.

### Release Notes

<!--
Optionally, replace `N/A` with text to be included in the next release
notes.
The `N/A` bullets are ignored. If you refer to one or more Positron
issues,
these issues are used to collect information about the feature or
bugfix, such
as the relevant language pack as determined by Github labels of type
`lang: `.
  The note will automatically be tagged with the language.

These notes are typically filled by the Positron team. If you are an
external
  contributor, you may ignore this section.
-->

#### New Features

- N/A

#### Bug Fixes

- N/A


### QA Notes

Windows, Linux and MacOS arm64 should have ipykernel available in
Positron python sessions, where as macOS x86 will still need ipykernel
to be installed on first use.

---------

Signed-off-by: Wasim Lorgat <[email protected]>
Co-authored-by: seem <[email protected]>

build/darwin/create-universal-app.ts

@@ -46,6 +46,8 @@ const stashPatterns = [
 	'**/html.icns',
 	// Exclusions from Python language pack (positron-python)
 	'**/pydevd/**',             // Cython pre-built binaries for Python debugging
+	'**/lib/ipykernel/**',      // Bundled IPyKernel dependencies
+	'**/lib/ipykernel/**/.dylibs/**',  // Bundled IPyKernel dependency dylibs
 	// Exclusions from R language pack (positron-r)
 	'**/ark',                   // Compiled R kernel and LSP
 	// Exclusions from Kallichore Jupyter supervisor

extensions/positron-python/gulpfile.js

@@ -32,6 +32,10 @@ const isCI = process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined
 
 // --- Start Positron ---
 const pythonCommand = locatePython();
+const arch = os.arch();
+if (arch !== 'x64' && arch !== 'arm64') {
+    throw new Error(`Unsupported architecture: ${arch}`);
+}
 // --- End Positron ---
 
 gulp.task('compileCore', (done) => {
@@ -279,43 +283,45 @@ async function vendorPythonKernelRequirements() {
     await spawnAsync(pythonCommand, ['scripts/vendor.py']);
 }
 
-async function installIPyKernelPurePythonRequirements() {
+async function bundleIPykernel() {
+    const pythonVersions = ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'];
+    const minimumPythonVersion = '3.8';
+
+    // Pure Python 3 requirements.
     await pipInstall([
         '--target',
         './python_files/lib/ipykernel/py3',
         '--implementation',
         'py',
         '--python-version',
-        '3.8',
+        minimumPythonVersion,
         '--abi',
         'none',
         '-r',
         './python_files/ipykernel_requirements/py3-requirements.txt',
     ]);
-}
 
-async function installIPyKernelCPythonVersionAgnosticRequirements() {
+    // CPython 3 requirements (specific to platform and architecture).
     await pipInstall([
         '--target',
-        './python_files/lib/ipykernel/cp3',
+        `./python_files/lib/ipykernel/${arch}/cp3`,
         '--implementation',
         'cp',
         '--python-version',
         '3.8',
         '--abi',
         'abi3',
         '-r',
-        './python_files/ipykernel_requirements/cp3-requirements.txt',
+        `./python_files/ipykernel_requirements/cp3-requirements.txt`,
     ]);
-}
 
-async function installIPyKernelCPythonVersionSpecificRequirements() {
-    for (const pythonVersion of ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']) {
+    // CPython 3.x requirements (specific to platform, architecture, and Python version).
+    for (const pythonVersion of pythonVersions) {
         const shortVersion = pythonVersion.replace('.', '');
         const abi = `cp${shortVersion}`;
         await pipInstall([
             '--target',
-            `./python_files/lib/ipykernel/${abi}`,
+            `./python_files/lib/ipykernel/${arch}/${abi}`,
             '--implementation',
             'cp',
             '--python-version',
@@ -328,12 +334,6 @@ async function installIPyKernelCPythonVersionSpecificRequirements() {
     }
 }
 
-async function bundleIPykernel() {
-    await installIPyKernelPurePythonRequirements();
-    await installIPyKernelCPythonVersionAgnosticRequirements();
-    await installIPyKernelCPythonVersionSpecificRequirements();
-}
-
 gulp.task(
     'installPythonLibs',
     // Run in parallel since vendoring rewrites imports which is somewhat CPU-bound.

extensions/positron-python/scripts/pip-compile-ipykernel.py

@@ -311,7 +311,7 @@ def write_output(
     main(
         requirement="ipykernel",
         python_versions=["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"],
-        output_dir=Path("./python_files/lib/ipykernel/"),
+        output_dir=Path("./python_files/ipykernel_requirements/"),
         max_rounds=10,
         cache_dir=CACHE_DIR,
     )

extensions/positron-python/src/client/positron/ipykernel.ts

@@ -3,25 +3,38 @@
  *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import * as os from 'os';
+import * as path from 'path';
 import * as vscode from 'vscode';
+import * as fs from '../common/platform/fs-paths';
 import { IServiceContainer } from '../ioc/types';
 import { PythonEnvironment } from '../pythonEnvironments/info';
 import { IWorkspaceService } from '../common/application/types';
 import { IPythonExecutionFactory } from '../common/process/types';
-import { traceVerbose } from '../logging';
+import { traceWarn } from '../logging';
+import { EXTENSION_ROOT_DIR } from '../constants';
+
+/** IPyKernel bundle information. */
+export interface IPykernelBundle {
+    /** If bundling is disabled, the reason for it. */
+    disabledReason?: string;
+
+    /** Paths to be appended to the PYTHONPATH environment variable in this order, if bundling is enabled. */
+    paths?: string[];
+}
 
 /**
- * Check if an interpreter should use the bundled ipykernel.
+ * Get the IPyKernel bundle for a given interpreter.
  *
  * @param interpreter The interpreter to check.
  * @param serviceContainer The service container to use for dependency injection.
  * @param resource The resource to scope setting to.
  */
-export async function shouldUseBundledIpykernel(
+export async function getIpykernelBundle(
     interpreter: PythonEnvironment,
     serviceContainer: IServiceContainer,
     resource?: vscode.Uri,
-): Promise<boolean> {
+): Promise<IPykernelBundle> {
     // Get the required services.
     const workspaceService = serviceContainer.get<IWorkspaceService>(IWorkspaceService);
     const pythonExecutionFactory = serviceContainer.get<IPythonExecutionFactory>(IPythonExecutionFactory);
@@ -31,16 +44,13 @@ export async function shouldUseBundledIpykernel(
         .getConfiguration('python', resource)
         .get<boolean>('useBundledIpykernel', true);
     if (!useBundledIpykernel) {
-        traceVerbose('createPythonRuntime: ipykernel bundling is disabled');
-        return false;
+        return { disabledReason: 'useBundledIpykernel setting is disabled' };
     }
-    traceVerbose('createPythonRuntime: ipykernel bundling is enabled, checking if interpreter is supported');
 
     // Check if ipykernel is bundled for the interpreter version.
     // (defined in scripts/pip-compile-ipykernel.py).
     if (interpreter.version?.major !== 3 || ![8, 9, 10, 11, 12, 13].includes(interpreter.version?.minor)) {
-        traceVerbose(`createPythonRuntime: ipykernel not bundled for interpreter version: ${interpreter.version?.raw}`);
-        return false;
+        return { disabledReason: `unsupported interpreter version: ${interpreter.version?.raw}` };
     }
 
     // Get the interpreter implementation if it's not already available.
@@ -53,12 +63,25 @@ export async function shouldUseBundledIpykernel(
     // Check if ipykernel is bundled for the interpreter implementation.
     // (defined in scripts/pip-compile-ipykernel.py).
     if (implementation !== 'cpython') {
-        traceVerbose(
-            `createPythonRuntime: ipykernel not bundled for interpreter implementation: ${interpreter.implementation}`,
-        );
-        return false;
+        return { disabledReason: `unsupported interpreter implementation: ${implementation}` };
+    }
+
+    // Append the bundle paths (defined in gulpfile.js) to the PYTHONPATH environment variable.
+    const arch = os.arch();
+    const cpxSpecifier = `cp${interpreter.version.major}${interpreter.version.minor}`;
+    const paths = [
+        path.join(EXTENSION_ROOT_DIR, 'python_files', 'lib', 'ipykernel', arch, cpxSpecifier),
+        path.join(EXTENSION_ROOT_DIR, 'python_files', 'lib', 'ipykernel', arch, 'cp3'),
+        path.join(EXTENSION_ROOT_DIR, 'python_files', 'lib', 'ipykernel', 'py3'),
+    ];
+
+    for (const path of paths) {
+        if (!(await fs.pathExists(path))) {
+            // This shouldn't happen. Did something go wrong during `npm install`?
+            traceWarn(`ipykernel bundle path does not exist: ${path}`);
+            return { disabledReason: `bundle path does not exist: ${path}` };
+        }
     }
 
-    traceVerbose(`createPythonRuntime: ipykernel bundling is supported by interpreter: ${interpreter.path}`);
-    return true;
+    return { paths };
 }

extensions/positron-python/src/client/positron/runtime.ts

@@ -18,11 +18,11 @@ import { IInstaller, Product, ProductInstallStatus } from '../common/types';
 import { IApplicationEnvironment, IWorkspaceService } from '../common/application/types';
 import { EXTENSION_ROOT_DIR, IPYKERNEL_VERSION, PYTHON_LANGUAGE } from '../common/constants';
 import { EnvLocationHeuristic, getEnvLocationHeuristic } from '../interpreter/configuration/environmentTypeComparer';
-import { shouldUseBundledIpykernel } from './ipykernel';
+import { getIpykernelBundle, IPykernelBundle } from './ipykernel';
 
 export interface PythonRuntimeExtraData {
     pythonPath: string;
-    useBundledIpykernel?: boolean;
+    ipykernelBundle?: IPykernelBundle;
 }
 
 export async function createPythonRuntimeMetadata(
@@ -43,20 +43,24 @@ export async function createPythonRuntimeMetadata(
     traceInfo('createPythonRuntime: getting extension runtime settings');
 
     // Check if we should use the bundled ipykernel.
-    const useBundledIpykernel = await shouldUseBundledIpykernel(interpreter, serviceContainer, workspaceUri);
+    const ipykernelBundle = await getIpykernelBundle(interpreter, serviceContainer, workspaceUri);
 
     // Determine if a compatible version of ipykernel is available (either bundled or already installed).
     let hasCompatibleKernel: boolean;
-    if (useBundledIpykernel) {
-        hasCompatibleKernel = true;
-    } else {
-        traceInfo('createPythonRuntime: checking if ipykernel is installed');
+    if (ipykernelBundle.disabledReason) {
+        traceInfo(
+            `createPythonRuntime: ipykernel bundling is disabled, ` +
+                `reason: ${ipykernelBundle.disabledReason}. ` +
+                `Checking if ipykernel is installed`,
+        );
         const productInstallStatus = await installer.isProductVersionCompatible(
             Product.ipykernel,
             IPYKERNEL_VERSION,
             interpreter,
         );
         hasCompatibleKernel = productInstallStatus === ProductInstallStatus.Installed;
+    } else {
+        hasCompatibleKernel = true;
     }
 
     // Define the startup behavior; request immediate startup if this is the
@@ -116,7 +120,7 @@ export async function createPythonRuntimeMetadata(
     // runtime session.
     const extraRuntimeData: PythonRuntimeExtraData = {
         pythonPath: interpreter.path,
-        useBundledIpykernel,
+        ipykernelBundle,
     };
 
     // Check the kernel supervisor's configuration; if it's  configured to