feat: allow opt-in for new backends

.github/requirements-capi.txt

@@ -0,0 +1,4 @@
+# Python packages for the GOPY_BACKEND=capi job in workflows/ci.yml.
+# pybindgen is deliberately absent: the capi backend must not need it.
+# used by the memory-leak checks on Windows, where the resource module is missing
+psutil

.github/requirements-cffi.txt

@@ -0,0 +1,5 @@
+# Python packages for the GOPY_BACKEND=cffi job in workflows/ci.yml.
+# pybindgen is deliberately absent: the cffi backend must not need it.
+cffi
+# used by the memory-leak checks on Windows, where the resource module is missing
+psutil

.github/requirements-nanobind.txt

@@ -0,0 +1,5 @@
+# Python packages for the GOPY_BACKEND=nanobind job in workflows/ci.yml.
+# pybindgen is deliberately absent: the nanobind backend must not need it.
+nanobind
+# used by the memory-leak checks on Windows, where the resource module is missing
+psutil

.github/requirements-pybind11.txt

@@ -0,0 +1,5 @@
+# Python packages for the GOPY_BACKEND=pybind11 job in workflows/ci.yml.
+# pybindgen is deliberately absent: the pybind11 backend must not need it.
+pybind11
+# used by the memory-leak checks on Windows, where the resource module is missing
+psutil

.github/workflows/ci.yml

@@ -14,23 +14,43 @@ env:
   GOTRACEBACK: crash
   GO111MODULE: auto
 
+# A new push to a pull request cancels its previous, now outdated, run.
+# Pushes to master always run to completion: each one adds to the coverage
+# and benchmark history.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
 
   build:
     name: Build (${{ matrix.platform }}, Go ${{ matrix.go-version }}, Python ${{ matrix.python-version }})
     strategy:
-      fail-fast: false
       matrix:
         # TODO: Consider official support matrix (OS and Go versions) and adjust this matrix accordingly
         go-version: [1.25.x, 1.24.x, 1.23.x, 1.22.x]
         platform: [ubuntu-latest, windows-latest, macos-15]
         python-version: ['3.11', '3.12']
     runs-on: ${{ matrix.platform }}
+    # A hung step fails here instead of at GitHub's 6h default.
+    timeout-minutes: 20
     env:
       # Coverage is measured on exactly one matrix leg, so the numbers stay
       # comparable from commit to commit.
       COVERAGE_LEG: ${{ matrix.platform == 'ubuntu-latest' && matrix.go-version == '1.25.x' && matrix.python-version == '3.12' }}
     steps:
+    # Real-time scanning inspects every file the compilers, linkers and pip
+    # write, which slows Windows jobs down.  Turned off first, so checkout and
+    # the Go/Python installs benefit too.  The status is printed before and
+    # after so the effect can be checked in the log.
+    - name: Disable Windows Defender real-time scanning
+      if: runner.os == 'Windows'
+      shell: pwsh
+      run: |
+        Get-MpComputerStatus | Select-Object RealTimeProtectionEnabled
+        Set-MpPreference -DisableRealtimeMonitoring $true
+        Get-MpComputerStatus | Select-Object RealTimeProtectionEnabled
+
     - name: Checkout code
       uses: actions/checkout@v4
 
@@ -48,26 +68,24 @@ jobs:
     - name: Install Linux packages
       if: matrix.platform == 'ubuntu-latest'
       run: |
-        sudo apt-get update
-        sudo apt-get install curl libffi-dev python3-cffi python3-pip
         # install pybindgen
         python3 -m pip install --user -U pybindgen
         # install goimports
-        go install golang.org/x/tools/cmd/goimports@latest
+        go install golang.org/x/tools/cmd/goimports@v0.29.0
 
     - name: Install macOS packages
       if: startsWith(matrix.platform, 'macos-')
       run: |
         python3 -m pip install -U pybindgen
-        go install golang.org/x/tools/cmd/goimports@latest
+        go install golang.org/x/tools/cmd/goimports@v0.29.0
 
     - name: Install Windows packages
       if: matrix.platform == 'windows-latest'
       run: |
         # install pybindgen and psutil (for memory tracking in tests)
         python -m pip install -U pybindgen psutil
         # install goimports
-        go install golang.org/x/tools/cmd/goimports@latest
+        go install golang.org/x/tools/cmd/goimports@v0.29.0
 
 
     - name: Build-Linux
@@ -132,3 +150,156 @@ jobs:
         disable_search: true
         token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: false
+
+  # Builds and tests each opt-in backend (GOPY_BACKEND), beside the main
+  # matrix and on one Go version.  None installs pybindgen: no backend here
+  # may need it.
+  #   cffi:     loads the cgo shim at runtime, with no C compiler step.
+  #   pybind11: compiles a C++ module against the same shim (see plainCShim in
+  #             bind/plainc.go), so needs no extra system packages: a C++
+  #             compiler is already required for CGO_ENABLED=1 everywhere.
+  #   nanobind: as pybind11 (see buildCXXModule in cmd_build.go), and also
+  #             compiles nanobind's own runtime (nb_combined.cpp) into every
+  #             module.
+  #   capi:     builds exactly as the default backend does, except that
+  #             build.py writes the C module itself (see bind/capi_build.py).
+  backends:
+    name: ${{ matrix.backend }} backend (${{ matrix.platform }}, Python ${{ matrix.python-version }})
+    strategy:
+      matrix:
+        backend: [cffi, pybind11, nanobind, capi]
+        platform: [ubuntu-latest, windows-latest, macos-15]
+        python-version: ['3.11', '3.12']
+        include:
+          # CXX defaults to "c++" in buildCXXModule (cmd_build.go), which
+          # isn't a recognized command on windows-latest's MinGW toolchain (the
+          # same one CGO_ENABLED=1 already needs there, so no extra install is
+          # needed).  timeout raises go test's own default (10m): each test
+          # here compiles twice (cgo, then a separate C++ step), and once
+          # measured at ~18.5s/test average on windows-latest, 32 tests alone
+          # used 590s of the default budget.
+          - backend: pybind11
+            cxx: g++
+            timeout: 30m
+          - backend: nanobind
+            cxx: g++
+            timeout: 30m
+    runs-on: ${{ matrix.platform }}
+    # Above the longest go test -timeout (30m), so that go test's own timeout,
+    # with its stack dump, fires first.
+    timeout-minutes: 40
+    env:
+      GOPY_BACKEND: ${{ matrix.backend }}
+      CXX: ${{ matrix.cxx }}
+      # print the python stack if the process crashes, e.g. at exit
+      PYTHONFAULTHANDLER: 1
+    steps:
+    # Real-time scanning inspects every file the compilers, linkers and pip
+    # write, which slows Windows jobs down.  Turned off first, so checkout and
+    # the Go/Python installs benefit too.  The status is printed before and
+    # after so the effect can be checked in the log.
+    - name: Disable Windows Defender real-time scanning
+      if: runner.os == 'Windows'
+      shell: pwsh
+      run: |
+        Get-MpComputerStatus | Select-Object RealTimeProtectionEnabled
+        Set-MpPreference -DisableRealtimeMonitoring $true
+        Get-MpComputerStatus | Select-Object RealTimeProtectionEnabled
+
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: .github/requirements-${{ matrix.backend }}.txt
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.25.x
+        cache: true
+
+    - name: Install packages
+      run: |
+        python -m pip install -r .github/requirements-${{ matrix.backend }}.txt
+        go install golang.org/x/tools/cmd/goimports@v0.29.0
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Test
+      run: go test -v -timeout=${{ matrix.timeout || '10m' }} ./...
+
+  # Compares per-call overhead across backends (see _examples/bench/run.sh):
+  # not a pass/fail check, just a table uploaded as a build artifact.  Runs
+  # on ubuntu-latest only -- the comparison is between backends, not OSes.
+  benchmark:
+    name: benchmark backends
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.25.x
+        cache: true
+
+    - name: Install packages
+      run: |
+        python -m pip install pybindgen cffi pybind11 nanobind
+        go install golang.org/x/tools/cmd/goimports@v0.29.0
+
+    - name: Run benchmark
+      run: |
+        bash _examples/bench/run.sh | tee benchmark.txt
+        python _examples/bench/bench_json.py < benchmark.txt > benchmark.json
+
+    - name: Upload benchmark table
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark
+        path: |
+          benchmark.txt
+          benchmark.json
+
+    # The results of earlier runs on master, kept in the Actions cache (no
+    # branch or token needed): every run restores the latest, a pull request
+    # from its base branch's.
+    - name: Restore benchmark history
+      uses: actions/cache/restore@v4
+      with:
+        path: benchmark-history
+        key: benchmark-history-${{ github.run_id }}
+        restore-keys: benchmark-history-
+
+    # Compares with the latest master run in the job summary, warning (but
+    # never failing, since CI runners' timings are noisy) at 2x slower, and
+    # adds this run to the history.
+    - name: Compare with earlier runs
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: gopy backends
+        tool: customSmallerIsBetter
+        output-file-path: benchmark.json
+        external-data-json-path: benchmark-history/data.json
+        summary-always: true
+        fail-on-alert: false
+
+    # Only master's runs make up the history.
+    - name: Save benchmark history
+      if: github.event_name == 'push'
+      uses: actions/cache/save@v4
+      with:
+        path: benchmark-history
+        key: benchmark-history-${{ github.run_id }}

.gitignore

@@ -1,9 +1,17 @@
-# IDE files
-.idea/
-*.iml
-
-# goreleaser build output
-/dist/
-
-# 'make cover' output
-/coverage.out
+# IDE files
+.idea/
+*.iml
+
+# goreleaser build output
+/dist/
+
+# python bytecode caches
+__pycache__/
+*.py[co]
+
+# built by TestBindCgoLDFlags
+/_examples/cgoldflags/clib/*.a
+
+# 'make cover' output
+/coverage.out
+/covdata/

Makefile

@@ -27,11 +27,17 @@ test:
 
 # Same as 'test', but records statement coverage of every gopy package
 # (-coverpkg), since most of bind/ is only exercised via main_test.go.
+# main_test.go runs gopy as a separate process (see TestMain), which
+# -coverprofile can't see, so the test binaries and gopy all write raw
+# coverage data to COVERDIR, merged into COVERPROFILE afterwards.
 COVERPROFILE=coverage.out
+COVERDIR=$(CURDIR)/covdata
 
 cover:
 	@echo "GO111MODULE = $(value GO111MODULE)"
-	$(GOTEST) -v -coverpkg=./... -coverprofile=$(COVERPROFILE) $(DIRS)
+	rm -rf $(COVERDIR) && mkdir -p $(COVERDIR)
+	$(GOTEST) -v -cover -coverpkg=./... $(DIRS) -args -test.gocoverdir=$(COVERDIR)
+	$(GOCMD) tool covdata textfmt -i=$(COVERDIR) -o=$(COVERPROFILE)
 	$(GOCMD) tool cover -func=$(COVERPROFILE) | tail -n 1
 
 clean: 

README.md

@@ -12,14 +12,14 @@ This is an improved version that works with current versions of Go (e.g., 1.15 -
 It also supports python modules having any number of Go packages, and generates a separate .py module file for each package, which link into a single common binding library.  It has been tested extensively on reproducing complex Go code in large libraries -- most stuff "just works".  For example, the [GoGi](https://github.com/goki/gi) GUI library is fully usable from python now (do `make; make install` in the python directory there, and try the `examples/widgets/widgets.py` demo).
 
 New features:
-* Callback methods from Go into Python now work: you can pass a python function to a Go function that has a function argument, and it will call the python function appropriately.
+* Callback methods from Go into Python now work: you can pass a python function to a Go function that has a function argument, and it will call the python function appropriately.  The Go function must only call it before it returns: don't keep it (in a variable, or a goroutine still running) and call it later.  The `cffi`, `pybind11` and `nanobind` backends refuse such a call, which then returns the zero value of its result type (e.g., `false` or `0`, which Go can't tell from a real answer; only `cffi` warns on stderr), and the other backends call a python function that may no longer exist.
 * The first embedded struct field (i.e., Go's version of type inheritance) is used to establish a corresponding class inheritance in the Python `class` wrappers, which then efficiently inherit all the methods, properties, etc.
 
 ## Installation
 
 Gopy now assumes that you are working with modules-based builds, and requires a valid `go.mod` file, and works only with Go versions 1.15 and above.
 
-Currently using [pybindgen](https://pybindgen.readthedocs.io/en/latest/tutorial/) to generate the low-level c-to-python bindings, but support for [cffi](https://cffi.readthedocs.io/en/latest/) should be relatively straightforward for those using PyPy instead of CPython (pybindgen should be significantly faster for CPython apparently).  You also need `goimports` to ensure the correct imports are included.
+By default, gopy uses [pybindgen](https://pybindgen.readthedocs.io/en/latest/tutorial/) to generate the low-level c-to-python bindings.  You also need `goimports` to ensure the correct imports are included.
 
 ```sh
 $ python3 -m pip install pybindgen
@@ -29,6 +29,22 @@ $ go install github.com/go-python/gopy@latest
 
 (This all assumes you have already installed [Go itself](https://golang.org/doc/install), and added `~/go/bin` to your `PATH`).
 
+### Choosing a backend (experimental)
+
+The `GOPY_BACKEND` environment variable picks a different tool for those bindings, for `gopy gen`, `gopy build` and `gopy pkg`:
+
+| `GOPY_BACKEND` | Needs | |
+|---|---|---|
+| `pybindgen` (default) | `pip install pybindgen` | |
+| `capi` | nothing beyond python itself | writes the C-API bindings directly, without pybindgen |
+| `cffi` | `pip install cffi` | |
+| `pybind11` | `pip install pybind11`, a C++ compiler | |
+| `nanobind` | `pip install nanobind`, a C++ compiler | |
+
+```sh
+$ GOPY_BACKEND=capi gopy build github.com/go-python/gopy/_examples/hi
+```
+
 To [install python modules](https://packaging.python.org/tutorials/packaging-projects/), you will need the python install packages:
 
 ```sh

SUPPORT_MATRIX.md

@@ -6,7 +6,9 @@ don't modify manually.
 Feature |py3
 --- | ---
 _examples/arrays | yes
+_examples/callbacks | yes
 _examples/cgo | yes
+_examples/cgoldflags | yes
 _examples/consts | yes
 _examples/cstrings | yes
 _examples/empty | yes

_examples/bench/bench.go

@@ -0,0 +1,23 @@
+// Copyright 2015-present, The go-python Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bench has a few trivial functions used to compare the per-call
+// overhead of gopy's backends (GOPY_BACKEND); see bench.py and the
+// "benchmark" job in .github/workflows/ci.yml.  Not part of the test suite
+// itself: no _examples/bench entry in main_test.go's features map.
+package bench
+
+// Add returns the sum of its arguments: the cheapest possible call, to
+// isolate per-call FFI overhead from any argument-marshaling cost.
+func Add(i, j int) int {
+	return i + j
+}
+
+// Concat concatenates two strings: a second data point, since string
+// arguments/returns cross the boundary very differently from an int
+// (a managed buffer + length or a null-terminated copy, depending on the
+// backend) and might not share the int case's relative cost.
+func Concat(a, b string) string {
+	return a + b
+}