From c340d7eeff9f28cc5a2cc8bb2f6b157c6127df12 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Tue, 24 Dec 2024 16:53:48 -0500 Subject: [PATCH 1/2] test: Skip build/install steps with nox's --no-install flag This saves time when you want to rerun the test suite with different pytest arguments but you haven't made any code changes in-between. --- .pre-commit-config.yaml | 2 +- noxfile.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b3f73e2bba..3353bf682fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: args: ["--pretty", "--show-error-codes"] additional_dependencies: [ 'keyring==24.2.0', - 'nox==2023.4.22', + 'nox==2024.03.02', 'pytest', 'types-docutils==0.20.0.3', 'types-setuptools==68.2.0.0', diff --git a/noxfile.py b/noxfile.py index 0d6192d1904..6e6e144bccb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,6 +19,7 @@ nox.options.reuse_existing_virtualenvs = True nox.options.sessions = ["lint"] +nox.needs_version = ">=2024.03.02" # for session.run_install() LOCATIONS = { "common-wheels": "tests/data/common_wheels", @@ -44,7 +45,9 @@ def run_with_protected_pip(session: nox.Session, *arguments: str) -> None: env = {"VIRTUAL_ENV": session.virtualenv.location} command = ("python", LOCATIONS["protected-pip"]) + arguments - session.run(*command, env=env, silent=True) + # By using run_install(), these installation steps can be skipped when -R + # or --no-install is passed. + session.run_install(*command, env=env, silent=True) def should_update_common_wheels() -> bool: @@ -84,8 +87,13 @@ def test(session: nox.Session) -> None: session.log(msg) # Build source distribution + # HACK: we want to skip building and installing pip when nox's --no-install + # flag is given (to save time when running tests back to back with different + # arguments), but unfortunately nox does not expose this configuration state + # yet. https://github.com/wntrblm/nox/issues/710 + no_install = "-R" in sys.argv or "--no-install" in sys.argv sdist_dir = os.path.join(session.virtualenv.location, "sdist") - if os.path.exists(sdist_dir): + if not no_install and os.path.exists(sdist_dir): shutil.rmtree(sdist_dir, ignore_errors=True) run_with_protected_pip(session, "install", "build") @@ -94,7 +102,7 @@ def test(session: nox.Session) -> None: # pip, so uninstall pip to force build to provision a known good version of pip. run_with_protected_pip(session, "uninstall", "pip", "-y") # fmt: off - session.run( + session.run_install( "python", "-I", "-m", "build", "--sdist", "--outdir", sdist_dir, silent=True, ) From 5ce1145511bc59422765998a5a039388a9068155 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Tue, 24 Dec 2024 16:57:15 -0500 Subject: [PATCH 2/2] CI: micro-optimize test collection & pass nox's --no-install Pytest can be pretty slow to collect pip's entire test suite and prepare for test execution. I've observed a ~15s delay from invoking pytest to the first test running in CI in the worst case. This can be improved by reducing how many files pytest has to process while collecting tests. In short, passing tests/unit is faster than -m unit. In addition, use nox's --no-install flag to skip redundant build and install steps on the 2nd nox session invocation (for the integration tests), which was made possible by the previous commit. --- .github/workflows/ci.yml | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2af281c7027..a946a3d3e36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,17 +149,17 @@ jobs: - name: Run unit tests run: >- nox -s test-${{ matrix.python.key || matrix.python }} -- - -m unit + tests/unit --verbose --numprocesses auto --showlocals - name: Run integration tests run: >- - nox -s test-${{ matrix.python.key || matrix.python }} -- - -m integration + nox -s test-${{ matrix.python.key || matrix.python }} --no-install -- + tests/functional --verbose --numprocesses auto --showlocals --durations=5 tests-windows: - name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group }} + name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group.number }} runs-on: ${{ matrix.os }}-latest needs: [packaging, determine-changes] @@ -180,7 +180,9 @@ jobs: # - "3.11" # - "3.12" - "3.13" - group: [1, 2] + group: + - { number: 1, pytest-filter: "not test_install" } + - { number: 2, pytest-filter: "test_install" } steps: - uses: actions/checkout@v4 @@ -198,29 +200,19 @@ jobs: TEMP: "C:\\Temp" # Main check - - name: Run unit tests - if: matrix.group == 1 + - name: Run unit tests (group 1) + if: matrix.group.number == 1 run: >- nox -s test-${{ matrix.python }} -- - -m unit + tests/unit --verbose --numprocesses auto --showlocals env: TEMP: "C:\\Temp" - - name: Run integration tests (group 1) - if: matrix.group == 1 + - name: Run integration tests (group ${{ matrix.group.number }}) run: >- - nox -s test-${{ matrix.python }} -- - -m integration -k "not test_install" - --verbose --numprocesses auto --showlocals - env: - TEMP: "C:\\Temp" - - - name: Run integration tests (group 2) - if: matrix.group == 2 - run: >- - nox -s test-${{ matrix.python }} -- - -m integration -k "test_install" + nox -s test-${{ matrix.python }} --no-install -- + tests/functional -k "${{ matrix.group.pytest-filter }}" --verbose --numprocesses auto --showlocals env: TEMP: "C:\\Temp" @@ -251,7 +243,7 @@ jobs: - name: Run integration tests run: >- nox -s test-3.10 -- - -m integration + tests/functional --verbose --numprocesses auto --showlocals --durations=5 --use-zipapp