Add an experimental pyenv-binary plugin with a save command (#3487)

* pyenv-binary: add experimental plugin skeleton

Adds a new plugin, decoupled from `pyenv install`, for packaging and installing
relocatable Python binaries. This commit is just the command dispatcher; the
individual subcommands follow.

`pyenv binary <command>` dispatches to the matching script under the plugin's
libexec, so subcommands stay out of the top-level `pyenv commands` list.

Groundwork for the binary distribution support discussed in #2334.

* pyenv-binary: add the `save` subcommand

`pyenv binary save <version> [<output-dir>]` packs an installed version into a
relocatable .tar.gz with relative paths and writes a metadata file recording the
build platform, distro, libc version and the external system libraries the
build links against (via `ldd` on Linux, `otool -L` on macOS).

* pyenv-binary: reject invalid version names and drop readlink -f in save

A version is a single directory name under versions/, so refuse names with a
slash or a dot-dot component before building the prefix path.

Also emit the python paths directly instead of `readlink -f`, which BSD
readlink on macOS does not support; ldd/otool follow the symlinks anyway, so
the resolved paths were never needed.

* pyenv-binary: test the `save` subcommand

Cover argument validation, the version-name guard, and packaging an
installed version into an archive with matching metadata.

* pyenv-binary: test dependency parsing and tighten the version guard

Add tests that feed realistic ldd and otool listings through save and
check only the libraries resolving outside the prefix end up in the
metadata, since that filtering is the fiddliest part of the command.

Narrow the version guard to reject `.' and `..' rather than any name
containing a dot-dot, now that a slash is already refused, and note why
the find still matches *.so.* even though CPython does not produce them.

plugins/.gitignore

@@ -2,4 +2,5 @@
 !/.gitignore
 !/version-ext-compat
 !/python-build
+!/pyenv-binary
 /python-build/test/build

plugins/pyenv-binary/README.md

@@ -0,0 +1,27 @@
+# pyenv-binary (experimental)
+
+Package an installed Python version into a relocatable archive that can be
+installed on another machine.
+
+This is experimental and intentionally decoupled: it does not change
+`pyenv install` or any other command. You drive it explicitly through
+`pyenv binary`. Run `pyenv binary <command> --help` for details on a command.
+
+## Portability
+
+An archive is portable across machines that share its build platform (OS,
+architecture and a compatible libc) and have the recorded system libraries. It
+is not portable across, say, glibc and musl, or to an older glibc; the platform
+and dependency metadata exist to catch that.
+
+## Commands
+
+### `pyenv binary save <version> [<output-dir>]`
+
+Packs an installed version into `<version>-<platform>.tar.gz` (relative paths)
+and writes `<version>-<platform>.meta` describing the build platform (OS, arch,
+distro and libc version) and the system libraries the build links against.
+
+```sh
+pyenv binary save 3.12.7 ./dist
+```

plugins/pyenv-binary/bin/pyenv-binary

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# Summary: Package an installed Python version as a relocatable binary (experimental)
+#
+# Usage: pyenv binary <command> [<args>]
+#
+# `pyenv binary` packages an already-installed Python version into a relocatable
+# archive that can be installed on another machine. It is experimental and does
+# not touch `pyenv install' or any other command.
+#
+# Run `pyenv binary' to list its commands, or `pyenv binary <command> --help'
+# for command-specific help.
+#
+set -e
+[ -n "$PYENV_DEBUG" ] && set -x
+
+libexec="${BASH_SOURCE%/*}/../libexec"
+
+list_commands() {
+  local path
+  for path in "$libexec"/pyenv-binary-*; do
+    [ -e "$path" ] && echo "${path##*/pyenv-binary-}"
+  done
+}
+
+# Provide pyenv completions
+if [ "$1" = "--complete" ]; then
+  shift
+  if [ -z "$1" ]; then
+    list_commands
+  else
+    command_path="${libexec}/pyenv-binary-$1"
+    shift
+    [ -x "$command_path" ] && exec "$command_path" --complete "$@"
+  fi
+  exit
+fi
+
+subcommand="$1"
+if [ -z "$subcommand" ]; then
+  { pyenv-help binary
+    echo
+    echo "Commands:"
+    list_commands | sed 's/^/   /'
+  } >&2
+  exit 1
+fi
+shift
+
+command_path="${libexec}/pyenv-binary-${subcommand}"
+if [ ! -x "$command_path" ]; then
+  echo "pyenv-binary: no such command \`${subcommand}'" >&2
+  exit 1
+fi
+
+if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+  # `pyenv help' looks the command up on PATH; the subcommands live in libexec.
+  PATH="${libexec}:${PATH}" exec pyenv-help "binary-${subcommand}"
+fi
+
+exec "$command_path" "$@"

plugins/pyenv-binary/libexec/pyenv-binary-save

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+#
+# Summary: Save an installed Python version as a relocatable archive
+#
+# Usage: pyenv binary save <version> [<output-dir>]
+#
+#   Packs an installed version into a relocatable .tar.gz (relative paths) and
+#   writes a metadata file listing the build platform and the system libraries
+#   it links against, so an installer can check compatibility before unpacking.
+#
+#   <version>      An installed version, as listed by `pyenv versions --bare'.
+#   <output-dir>   Where to write the archive and metadata (default: `.').
+#
+set -e
+[ -n "$PYENV_DEBUG" ] && set -x
+
+# Provide pyenv completions
+if [ "$1" = "--complete" ]; then
+  exec pyenv-versions --bare
+fi
+
+version="$1"
+output_dir="${2:-$PWD}"
+
+if [ -z "$version" ]; then
+  echo "Usage: pyenv binary save <version> [<output-dir>]" >&2
+  exit 1
+fi
+
+# A version is a single directory name under versions/. With no slash allowed,
+# the only remaining names that could point elsewhere are `.' and `..'.
+case "$version" in
+*/* | .. | . )
+  echo "pyenv-binary: invalid version name \`${version}'" >&2
+  exit 1
+  ;;
+esac
+
+prefix="${PYENV_ROOT}/versions/${version}"
+if [ ! -d "${prefix}/bin" ]; then
+  echo "pyenv-binary: version \`${version}' is not installed" >&2
+  exit 1
+fi
+
+os="$(uname -s)"
+arch="$(uname -m)"
+platform="$(printf '%s' "$os" | tr '[:upper:]' '[:lower:]')-${arch}"
+
+# Record the distro and libc version. Platform and arch alone are too coarse to
+# judge compatibility: a build is only portable to a matching libc (e.g. a
+# glibc 2.36 build will not load on an older glibc, nor on musl at all).
+distro=""
+libc=""
+if [ -r /etc/os-release ]; then
+  distro="$( . /etc/os-release && printf '%s %s' "${ID:-}" "${VERSION_ID:-}" )"
+elif [ "$os" = "Darwin" ]; then
+  distro="macos $(sw_vers -productVersion 2>/dev/null)"
+fi
+if [ "$os" = "Linux" ]; then
+  libc="$(getconf GNU_LIBC_VERSION 2>/dev/null || true)"
+fi
+
+# List the external shared libraries the install links against: those that
+# resolve outside its own prefix, so they must already exist on the target.
+# The interpreter plus every bundled shared object are inspected.
+system_deps() {
+  local f
+  {
+    for f in "${prefix}"/bin/python*; do
+      [ -e "$f" ] && printf '%s\n' "$f"
+    done
+    # CPython ships its extension modules as *.so (and *.dylib on macOS). A
+    # bare *.so.* is unusual for CPython itself, but the odd build carries a
+    # versioned copy alongside, so match it too rather than miss a dependency.
+    find "${prefix}" -type f \( -name '*.so' -o -name '*.so.*' -o -name '*.dylib' \)
+  } | sort -u | while IFS= read -r f; do
+    if [ "$os" = "Darwin" ]; then
+      otool -L "$f" 2>/dev/null | tail -n +2 | awk -v pfx="${prefix}/" \
+        '$1 !~ /^@/ && substr($1, 1, length(pfx)) != pfx { print $1 }'
+    else
+      ldd "$f" 2>/dev/null | awk -v pfx="${prefix}/" \
+        '$2 == "=>" && $3 ~ /^\// && substr($3, 1, length(pfx)) != pfx { print $1 }'
+    fi
+  done | sort -u
+}
+
+mkdir -p "$output_dir"
+archive="${version}-${platform}.tar.gz"
+metadata="${version}-${platform}.meta"
+
+tar -C "$prefix" -czf "${output_dir}/${archive}" .
+
+{
+  echo "# pyenv-binary metadata"
+  echo "version=${version}"
+  echo "os=${os}"
+  echo "arch=${arch}"
+  echo "platform=${platform}"
+  [ -n "$distro" ] && echo "distro=${distro}"
+  [ -n "$libc" ] && echo "libc=${libc}"
+  echo "archive=${archive}"
+  system_deps | while IFS= read -r dep; do
+    [ -n "$dep" ] && echo "dep=${dep}"
+  done
+} > "${output_dir}/${metadata}"
+
+echo "Saved ${archive} and ${metadata} to ${output_dir}"

plugins/pyenv-binary/test/save.bats

@@ -0,0 +1,117 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+create_version() {
+  mkdir -p "${PYENV_ROOT}/versions/$1/bin"
+}
+
+platform() {
+  echo "$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+}
+
+@test "fails with no version given" {
+  run pyenv-binary-save
+  assert_failure "Usage: pyenv binary save <version> [<output-dir>]"
+}
+
+@test "fails for a version that is not installed" {
+  run pyenv-binary-save 9.9.9
+  assert_failure "pyenv-binary: version \`9.9.9' is not installed"
+}
+
+@test "rejects a version name containing a slash" {
+  run pyenv-binary-save "foo/bar"
+  assert_failure "pyenv-binary: invalid version name \`foo/bar'"
+}
+
+@test "rejects the parent directory reference" {
+  run pyenv-binary-save ".."
+  assert_failure "pyenv-binary: invalid version name \`..'"
+}
+
+@test "packages an installed version" {
+  create_version "3.12.7"
+  local out="${BATS_TEST_TMPDIR}/dist"
+  run pyenv-binary-save "3.12.7" "$out"
+  assert_success "Saved 3.12.7-$(platform).tar.gz and 3.12.7-$(platform).meta to $out"
+  assert [ -f "${out}/3.12.7-$(platform).tar.gz" ]
+  assert [ -f "${out}/3.12.7-$(platform).meta" ]
+}
+
+@test "records the platform in the metadata" {
+  create_version "3.12.7"
+  local out="${BATS_TEST_TMPDIR}/dist"
+  pyenv-binary-save "3.12.7" "$out" >/dev/null
+  run cat "${out}/3.12.7-$(platform).meta"
+  assert_success
+  assert_output_contains "version=3.12.7"
+  assert_output_contains "platform=$(platform)"
+  assert_output_contains "archive=3.12.7-$(platform).tar.gz"
+}
+
+# Put an executable on PATH that stands in for a system tool during the test.
+stub() {
+  local name="$1"
+  local dir="${BATS_TEST_TMPDIR}/stubs"
+  mkdir -p "$dir"
+  { echo "#!/usr/bin/env bash"; cat -; } > "${dir}/${name}"
+  chmod +x "${dir}/${name}"
+  export PATH="${dir}:$PATH"
+}
+
+@test "records only the libraries ldd resolves outside the prefix" {
+  create_version "3.12.7"
+  touch "${PYENV_ROOT}/versions/3.12.7/bin/python3.12"
+
+  # A realistic ldd listing: the vdso and the loader have no `=>' mapping,
+  # libpython resolves inside the prefix, and libc/libm are external.
+  stub ldd <<'STUB'
+prefix="${PYENV_ROOT}/versions/3.12.7"
+cat <<EOF
+	linux-vdso.so.1 (0x00007ffd1adfe000)
+	libpython3.12.so.1.0 => ${prefix}/lib/libpython3.12.so.1.0 (0x00007f4a3c000000)
+	libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f4a3bc00000)
+	libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f4a3b800000)
+	/lib64/ld-linux-x86-64.so.2 (0x00007f4a3c200000)
+EOF
+STUB
+
+  run pyenv-binary-save "3.12.7" "${BATS_TEST_TMPDIR}/dist"
+  assert_success
+
+  run grep '^dep=' "${BATS_TEST_TMPDIR}/dist/"*.meta
+  assert_output "dep=libc.so.6
+dep=libm.so.6"
+}
+
+@test "records only the libraries otool resolves outside the prefix" {
+  create_version "3.12.7"
+  touch "${PYENV_ROOT}/versions/3.12.7/bin/python3.12"
+
+  # Take the macOS path by faking the platform, then feed a realistic otool
+  # listing: the first line names the file, @rpath and in-prefix entries are
+  # bundled, and libSystem is the one external dependency.
+  stub uname <<'STUB'
+case "$1" in
+  -s) echo Darwin ;;
+  -m) echo arm64 ;;
+  *) exec /usr/bin/uname "$@" ;;
+esac
+STUB
+  stub otool <<'STUB'
+prefix="${PYENV_ROOT}/versions/3.12.7"
+cat <<EOF
+${2}:
+	@rpath/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)
+	${prefix}/lib/libcrypto.3.dylib (compatibility version 3.0.0, current version 3.0.0)
+	/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1345.0.0)
+EOF
+STUB
+
+  run pyenv-binary-save "3.12.7" "${BATS_TEST_TMPDIR}/dist"
+  assert_success
+
+  run grep '^dep=' "${BATS_TEST_TMPDIR}/dist/"*.meta
+  assert_output "dep=/usr/lib/libSystem.B.dylib"
+}

plugins/pyenv-binary/test/test_helper.bash

@@ -0,0 +1,68 @@
+setup() {
+  export PYENV_ROOT="${BATS_TEST_TMPDIR}/root"
+  PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+  PATH="${BATS_TEST_DIRNAME}/../libexec:$PATH"
+  export PATH
+
+  # If test specific setup exist, run it
+  if [[ $(type -t _setup) == function ]]; then
+    _setup
+  fi
+}
+
+flunk() {
+  { if [ "$#" -eq 0 ]; then cat -
+    else echo "$@"
+    fi
+  } | sed "s:${BATS_TEST_TMPDIR}:\${BATS_TEST_TMPDIR}:g" >&2
+  return 1
+}
+
+assert() {
+  if ! "$@"; then
+    flunk "failed: $@"
+  fi
+}
+
+assert_success() {
+  if [ "$status" -ne 0 ]; then
+    { echo "command failed with exit status $status"
+      echo "output: $output"
+    } | flunk
+  elif [ "$#" -gt 0 ]; then
+    assert_output "$1"
+  fi
+}
+
+assert_failure() {
+  if [ "$status" -eq 0 ]; then
+    flunk "expected failed exit status"
+  elif [ "$#" -gt 0 ]; then
+    assert_output "$1"
+  fi
+}
+
+assert_equal() {
+  if [ "$1" != "$2" ]; then
+    { echo "expected: $1"
+      echo "actual:   $2"
+    } | flunk
+  fi
+}
+
+assert_output() {
+  local expected
+  if [ $# -eq 0 ]; then expected="$(cat -)"
+  else expected="$1"
+  fi
+  assert_equal "$expected" "$output"
+}
+
+assert_output_contains() {
+  local expected="$1"
+  echo "$output" | grep -F "$expected" >/dev/null || {
+    { echo "expected output to contain: $expected"
+      echo "actual: $output"
+    } | flunk
+  }
+}