diff --git a/.ci/build.sh b/.ci/build.sh
index ae10cb67155..c172e513f68 100755
--- a/.ci/build.sh
+++ b/.ci/build.sh
@@ -2,6 +2,5 @@
 
 set -e
 
-python3 -m coverage erase
 make clean
 make install-coverage
diff --git a/.ci/install.sh b/.ci/install.sh
index cd2bf93931f..a5fd9d265ea 100755
--- a/.ci/install.sh
+++ b/.ci/install.sh
@@ -24,13 +24,7 @@ sudo apt-get -qq install libfreetype6-dev liblcms2-dev libtiff-dev python3-tk\
 
 python3 -m pip install --upgrade pip
 python3 -m pip install --upgrade wheel
-python3 -m pip install coverage
-python3 -m pip install defusedxml
 python3 -m pip install ipython
-python3 -m pip install olefile
-python3 -m pip install -U pytest
-python3 -m pip install -U pytest-cov
-python3 -m pip install -U pytest-timeout
 # optional test dependencies, only install if there's a binary package.
 python3 -m pip install --only-binary=:all: numpy || true
 python3 -m pip install --only-binary=:all: pyarrow || true
diff --git a/.ci/test.sh b/.ci/test.sh
index 87a605d84be..fb03223db6d 100755
--- a/.ci/test.sh
+++ b/.ci/test.sh
@@ -4,4 +4,5 @@ set -e
 
 python3 -c "from PIL import Image"
 
-python3 -bb -m pytest -vv -x -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests $REVERSE
+python3 -bb -m pytest -vv -x -W always Tests -m "isolated" -n0
+python3 -bb -m pytest -vv -x -W always Tests -m "not isolated" --cov PIL --cov Tests --cov-report term --cov-report xml $REVERSE
diff --git a/.github/workflows/test-mingw.yml b/.github/workflows/test-mingw.yml
index 6bb88681907..07bdb1793b8 100644
--- a/.github/workflows/test-mingw.yml
+++ b/.github/workflows/test-mingw.yml
@@ -73,7 +73,7 @@ jobs:
           pushd depends && ./install_extra_test_images.sh && popd
 
       - name: Build Pillow
-        run: CFLAGS="-coverage" python3 -m pip install .
+        run: CFLAGS="-coverage" python3 -m pip install .[tests]
 
       - name: Test Pillow
         run: |
diff --git a/.github/workflows/test-valgrind-memory.yml b/.github/workflows/test-valgrind-memory.yml
index fbc4e978cf7..05d5f1972a8 100644
--- a/.github/workflows/test-valgrind-memory.yml
+++ b/.github/workflows/test-valgrind-memory.yml
@@ -61,5 +61,12 @@ jobs:
       run: |
         # The Pillow user in the docker container is UID 1001
         sudo chown -R 1001 $GITHUB_WORKSPACE
-        docker run --name pillow_container -e "PILLOW_VALGRIND_TEST=true" -e "SPLIT_COUNT=4" -e "SPLIT_INDEX=${{ matrix.split }}" -v $GITHUB_WORKSPACE:/Pillow pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }} /Pillow/depends/docker-test-valgrind-memory.sh
+        docker run --name pillow_container \
+          -e "PILLOW_VALGRIND_TEST=true" \
+          -e "SPLIT_COUNT=4" \
+          -e "SPLIT_INDEX=${{ matrix.split }}" \
+          -e "PYTEST_ADDOPTS=-n0" \
+          -v $GITHUB_WORKSPACE:/Pillow \
+          pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }} \
+          bash -c "python3 -m pip install pytest-xdist && /Pillow/depends/docker-test-valgrind-memory.sh"
         sudo chown -R runner $GITHUB_WORKSPACE
diff --git a/.github/workflows/test-valgrind.yml b/.github/workflows/test-valgrind.yml
index 444f71d323a..ea13c5519c4 100644
--- a/.github/workflows/test-valgrind.yml
+++ b/.github/workflows/test-valgrind.yml
@@ -55,5 +55,11 @@ jobs:
       run: |
         # The Pillow user in the docker container is UID 1001
         sudo chown -R 1001 $GITHUB_WORKSPACE
-        docker run --name pillow_container -e "PILLOW_VALGRIND_TEST=true" -e "SPLIT_COUNT=4" -e "SPLIT_INDEX=${{ matrix.split }}" -v $GITHUB_WORKSPACE:/Pillow pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }}
+        docker run --name pillow_container \
+          -e "PILLOW_VALGRIND_TEST=true" \
+          -e "SPLIT_COUNT=4" \
+          -e "SPLIT_INDEX=${{ matrix.split }}" \
+          -e "PYTEST_ADDOPTS=-n0" \
+          -v $GITHUB_WORKSPACE:/Pillow \
+          pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }}
         sudo chown -R runner $GITHUB_WORKSPACE
diff --git a/Makefile b/Makefile
index ba8ccd095f4..b1f4f4699ea 100644
--- a/Makefile
+++ b/Makefile
@@ -62,7 +62,7 @@ install:
 
 .PHONY: install-coverage
 install-coverage:
-	CFLAGS="-coverage -Werror=implicit-function-declaration" python3 -m pip -v install .
+	CFLAGS="-coverage -Werror=implicit-function-declaration" python3 -m pip -v install .[tests]
 	python3 selftest.py
 
 .PHONY: debug
@@ -95,12 +95,6 @@ test:
 	python3 -c "import pytest" > /dev/null 2>&1 || python3 -m pip install pytest
 	python3 -m pytest -qq
 
-.PHONY: test-p
-test-p:
-	python3 -c "import xdist" > /dev/null 2>&1 || python3 -m pip install pytest-xdist
-	python3 -m pytest -qq -n auto
-
-
 .PHONY: valgrind
 valgrind:
 	python3 -c "import pytest_valgrind" > /dev/null 2>&1 || python3 -m pip install pytest-valgrind
diff --git a/Tests/helper.py b/Tests/helper.py
index 1ede4b7d5b1..51b06db7d73 100644
--- a/Tests/helper.py
+++ b/Tests/helper.py
@@ -211,6 +211,7 @@ def is_pypy() -> bool:
     return sys.implementation.name == "pypy"
 
 
+@pytest.mark.isolated
 @pytest.mark.skipif(sys.platform.startswith("win32"), reason="Requires Unix or macOS")
 # Per https://stackoverflow.com/a/29007723/51685, due to JIT compilation,
 # RSS utilization is known to grow in PyPy.
diff --git a/pyproject.toml b/pyproject.toml
index 16d82d46711..3d696b9a550 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,6 +70,7 @@ optional-dependencies.tests = [
   "packaging",
   "pytest",
   "pytest-cov",
+  "pytest-sugar",
   "pytest-timeout",
   "pytest-xdist",
   "setuptools",
@@ -206,7 +207,10 @@ pretty = true
 num_workers = 4
 
 [tool.pytest]
-addopts = [ "-ra", "--color=auto" ]
+addopts = [ "-ra", "--color=auto", "--numprocesses=logical", "--dist=worksteal" ]
+markers = [
+  "isolated: tests that are not compatible with pytest-xdist (deselect with '-m \"not isolated\"')",
+]
 testpaths = [
   "Tests",
 ]