From 371d658d60e9afe4cf4621cd108203d6887e2673 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 21 Aug 2026 16:53:32 -0400 Subject: [PATCH 1/3] Upgrade CI and Docker to Python 3.14 --- .github/workflows/codeql-analysis.yml | 21 ++- .github/workflows/create-release.yml | 140 +++++++++---------- .github/workflows/cron-conda.yml | 118 ++++++++-------- .github/workflows/deploy-pages.yml | 6 +- .github/workflows/docker-image.yml | 70 +++++----- Pipfile | 5 +- conda/requirements.txt | 26 ++-- darc/__init__.py | 2 +- docker/Dockerfile | 183 +++++++++--------------- docker/debug.dockerfile | 191 ++++++++------------------ docs/source/conf.py | 2 +- requirements.txt | 75 ++++------ setup.py | 2 +- util/conda-dist.py | 28 +++- 14 files changed, 352 insertions(+), 517 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 23a8111f..d28814f3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,23 +13,18 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write steps: - name: Checkout repository - uses: actions/checkout@v4 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} + uses: actions/checkout@v7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v4 # Override language selection by uncommenting this and choosing your languages # with: # languages: go, javascript, csharp, python, cpp, java @@ -37,7 +32,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v4 # â„šī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -51,4 +46,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 60c2f9ed..5131252f 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -12,19 +12,21 @@ jobs: version_check: name: Check Version runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v7 with: - python-version: '3.12' + python-version: '3.14' - name: Install and Setup run: | - python -m pip install -U pip setuptools wheel + python -m pip install -U pip setuptools wheel packaging - name: Get Version id: get_version @@ -37,10 +39,10 @@ jobs: python -m pip install -e . echo "DARC_VERSION=$(python -c 'import darc; print(darc.__version__)')" >> $GITHUB_OUTPUT - echo "DARC_PRERELEASE=$(python -c 'import darc, pkg_resources; print(str(pkg_resources.parse_version(darc.__version__).is_prerelease).lower())')" >> $GITHUB_OUTPUT - echo "DARC_CONDA_LABEL=$(python -c 'import darc, pkg_resources; print("dev" if pkg_resources.parse_version(darc.__version__).is_prerelease else "main")')" >> $GITHUB_OUTPUT + echo "DARC_PRERELEASE=$(python -c 'from packaging.version import Version; import darc; print(str(Version(darc.__version__).is_prerelease).lower())')" >> $GITHUB_OUTPUT + echo "DARC_CONDA_LABEL=$(python -c 'from packaging.version import Version; import darc; print("dev" if Version(darc.__version__).is_prerelease else "main")')" >> $GITHUB_OUTPUT - - uses: mukunku/tag-exists-action@v1.2.0 + - uses: mukunku/tag-exists-action@v1.7.0 id: check_tag with: tag: "v${{ steps.get_version.outputs.DARC_VERSION }}" @@ -54,11 +56,13 @@ jobs: github: name: GitHub Release runs-on: ubuntu-latest + permissions: + contents: write needs: [ version_check ] if: ${{ startsWith(github.ref_name, 'v') || needs.version_check.outputs.DARC_TAG_EXISTS == 'false' }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: # ref: v${{ needs.version_check.outputs.DARC_VERSION }} fetch-depth: 0 @@ -71,7 +75,7 @@ jobs: - name: Create Release id: create_release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v3.0.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token with: @@ -91,11 +95,13 @@ jobs: tag: name: Conda Tag runs-on: ubuntu-latest + permissions: + contents: write needs: [ version_check ] if: ${{ startsWith(github.ref_name, 'v') || needs.version_check.outputs.DARC_TAG_EXISTS == 'false' }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: token: ${{ secrets.DARC }} # ref: v${{ needs.version_check.outputs.DARC_VERSION }} @@ -155,22 +161,15 @@ jobs: strategy: matrix: python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "pypy3.8" - - "pypy3.9" - - "pypy3.10" + - "3.14" steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: ref: v${{ needs.version_check.outputs.DARC_VERSION }} fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -191,7 +190,7 @@ jobs: python util/wheel_rename.py dist/*.whl - name: No Source Distribution for Python ${{ matrix.python-version }} - if: ${{ matrix.python-version != '3.12' }} + if: ${{ matrix.python-version != '3.14' }} run: | rm dist/*.tar.gz @@ -209,7 +208,7 @@ jobs: skip-existing: true - name: Upload package to GitHub Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v3.0.2 with: # allowUpdates: true files: | @@ -229,18 +228,14 @@ jobs: - ubuntu-latest # - windows-latest # skip for now python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" + - "3.14" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: ref: conda-${{ needs.version_check.outputs.DARC_VERSION }}+0 fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -275,65 +270,58 @@ jobs: python -m pip download -r conda/requirements.txt -d conda/wheels - name: Conda environment creation and activation - uses: conda-incubator/setup-miniconda@v3 - env: - DARC_VERSION: ${{ needs.version_check.outputs.DARC_VERSION }} - DARC_BUILD: "0" + uses: conda-incubator/setup-miniconda@v4.0.1 with: python-version: ${{ matrix.python-version }} - environment-file: conda/conda-build.yaml # Path to the build conda environment + activate-environment: base auto-update-conda: false - auto-activate-base: false + auto-activate: true + channels: conda-forge,defaults + conda-remove-defaults: false show-channel-urls: true - - name: Build and upload the conda packages (macOS) - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - if: ${{ matrix.os == 'macos-latest' }} - env: - DARC_VERSION: ${{ needs.version_check.outputs.DARC_VERSION }} - DARC_BUILD: "0" - with: - meta_yaml_dir: conda/darc - python-version: ${{ matrix.python-version }} # Values previously defined in `matrix` - platform_osx-64: true - user: jarryshaw - label: ${{ needs.version_check.outputs.DARC_CONDA_LABEL }} - token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret - - - name: Build and upload the conda packages (Ubuntu) - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - if: ${{ matrix.os == 'ubuntu-latest' }} - env: - DARC_VERSION: ${{ needs.version_check.outputs.DARC_VERSION }} - DARC_BUILD: "0" - with: - meta_yaml_dir: conda/darc - python-version: ${{ matrix.python-version }} # Values previously defined in `matrix` - platform_linux-64: true - user: jarryshaw - label: ${{ needs.version_check.outputs.DARC_CONDA_LABEL }} - token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret - - - name: Build and upload the conda packages (Windows) - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - if: ${{ matrix.os == 'windows-latest' }} + - name: Install conda build tools + run: | + conda install -n base --override-channels -c conda-forge -y conda-build + conda build --version + + - name: Build conda packages + id: conda_build + shell: bash -el {0} env: DARC_VERSION: ${{ needs.version_check.outputs.DARC_VERSION }} DARC_BUILD: "0" + working-directory: conda/darc + run: | + set -euo pipefail + + out_dir="$(mktemp -d -t compilation-XXXXXXXXXX)" + echo "out_dir=$out_dir" >> "$GITHUB_OUTPUT" + conda build . --package-format .tar.bz2 --no-anaconda-upload --output-folder "$out_dir" --python "${{ matrix.python-version }}" + package_paths="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort)" + if [ -z "$package_paths" ]; then + echo "::error::No conda package was built" + exit 1 + fi + printf '%s\n' "$package_paths" + { + echo 'packages<> "$GITHUB_OUTPUT" + + - name: Upload conda packages to Anaconda + uses: anaconda/actions/upload-package@v0.3.1 with: - meta_yaml_dir: conda/darc - python-version: ${{ matrix.python-version }} # Values previously defined in `matrix` - platform_win-64: true - user: jarryshaw - label: ${{ needs.version_check.outputs.DARC_CONDA_LABEL }} - token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret + token: ${{ secrets.ANACONDA_TOKEN }} + channel: jarryshaw + packages: ${{ steps.conda_build.outputs.packages }} + labels: ${{ needs.version_check.outputs.DARC_CONDA_LABEL }} - name: Upload package to GitHub Release if: ${{ matrix.os == 'ubuntu-latest' }} - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v3.0.2 with: - # allowUpdates: true - files: | - /tmp/compilation-*/**/*.tar.bz2 + files: ${{ steps.conda_build.outputs.packages }} tag_name: "v${{ needs.version_check.outputs.DARC_VERSION }}" token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index 7b1f9057..fedccdeb 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -10,14 +10,16 @@ jobs: conda-update: name: Update requirements.txt runs-on: ubuntu-latest + permissions: + contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v7 with: - python-version: '3.12' + python-version: '3.14' - name: Install and Setup run: | @@ -70,7 +72,7 @@ jobs: export TOR_PASS="null" echo "DARC_VERSION=$(python -c 'import darc; print(darc.__version__)')" >> $GITHUB_OUTPUT - echo "DARC_CONDA_LABEL=$(python -c 'import darc, pkg_resources; print("dev" if pkg_resources.parse_version(darc.__version__).is_prerelease else "main")')" >> $GITHUB_OUTPUT + echo "DARC_CONDA_LABEL=$(python -c 'from packaging.version import Version; import darc; print("dev" if Version(darc.__version__).is_prerelease else "main")')" >> $GITHUB_OUTPUT echo "DARC_BUILD=$(cat conda/build)" >> $GITHUB_OUTPUT outputs: @@ -84,9 +86,11 @@ jobs: runs-on: ubuntu-latest needs: [ conda-update ] if: needs.conda-update.outputs.CONDA_CHANGED == 'true' + permissions: + contents: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: token: ${{ secrets.DARC }} fetch-depth: 0 @@ -113,6 +117,8 @@ jobs: runs-on: ${{ matrix.os }} needs: [ conda-tag, conda-update ] if: needs.conda-update.outputs.CONDA_CHANGED == 'true' + permissions: + contents: write strategy: matrix: os: @@ -120,18 +126,14 @@ jobs: - ubuntu-latest # - windows-latest # skip for now python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" + - "3.14" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: ref: conda-${{ needs.conda-update.outputs.DARC_VERSION }}+${{ needs.conda-update.outputs.DARC_BUILD }} fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -166,68 +168,62 @@ jobs: python -m pip download -r conda/requirements.txt -d conda/wheels - name: Conda environment creation and activation - uses: conda-incubator/setup-miniconda@v3 - env: - DARC_VERSION: ${{ needs.conda-update.outputs.DARC_VERSION }} - DARC_BUILD: ${{ needs.conda-update.outputs.DARC_BUILD }} + uses: conda-incubator/setup-miniconda@v4.0.1 with: python-version: ${{ matrix.python-version }} - environment-file: conda/conda-build.yaml # Path to the build conda environment + activate-environment: base auto-update-conda: false - auto-activate-base: false + auto-activate: true + channels: conda-forge,defaults + conda-remove-defaults: false show-channel-urls: true - - name: Build and upload the conda packages (macOS) - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - if: ${{ matrix.os == 'macos-latest' }} - env: - DARC_VERSION: ${{ needs.conda-update.outputs.DARC_VERSION }} - DARC_BUILD: ${{ needs.conda-update.outputs.DARC_BUILD }} - with: - meta_yaml_dir: conda/darc - python-version: ${{ matrix.python-version }} # Values previously defined in `matrix` - platform_osx-64: true - overwrite: true - user: jarryshaw - label: ${{ needs.conda-update.outputs.DARC_CONDA_LABEL }} - token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret - - - name: Build and upload the conda packages (Ubuntu) - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - if: ${{ matrix.os == 'ubuntu-latest' }} - env: - DARC_VERSION: ${{ needs.conda-update.outputs.DARC_VERSION }} - DARC_BUILD: ${{ needs.conda-update.outputs.DARC_BUILD }} - with: - meta_yaml_dir: conda/darc - python-version: ${{ matrix.python-version }} # Values previously defined in `matrix` - platform_linux-64: true - overwrite: true - user: jarryshaw - label: ${{ needs.conda-update.outputs.DARC_CONDA_LABEL }} - token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret - - - name: Build and upload the conda packages (Windows) - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - if: ${{ matrix.os == 'windows-latest' }} + - name: Install conda build tools + run: | + conda install -n base --override-channels -c conda-forge -y conda-build + conda build --version + + - name: Build conda packages + id: conda_build + shell: bash -el {0} env: DARC_VERSION: ${{ needs.conda-update.outputs.DARC_VERSION }} DARC_BUILD: ${{ needs.conda-update.outputs.DARC_BUILD }} + working-directory: conda/darc + run: | + set -euo pipefail + + out_dir="$(mktemp -d -t compilation-XXXXXXXXXX)" + echo "out_dir=$out_dir" >> "$GITHUB_OUTPUT" + conda build . --package-format .tar.bz2 --no-anaconda-upload --output-folder "$out_dir" --python "${{ matrix.python-version }}" + package_paths="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort)" + if [ -z "$package_paths" ]; then + echo "::error::No conda package was built" + exit 1 + fi + printf '%s\n' "$package_paths" + package_paths_csv="$(printf '%s\n' "$package_paths" | paste -sd ',' -)" + echo "packages_csv=$package_paths_csv" >> "$GITHUB_OUTPUT" + { + echo 'packages<> "$GITHUB_OUTPUT" + + - name: Upload conda packages to Anaconda + uses: anaconda/actions/upload-package@v0.3.1 with: - meta_yaml_dir: conda/darc - python-version: ${{ matrix.python-version }} # Values previously defined in `matrix` - platform_win-64: true - overwrite: true - user: jarryshaw - label: ${{ needs.conda-update.outputs.DARC_CONDA_LABEL }} - token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret + token: ${{ secrets.ANACONDA_TOKEN }} + channel: jarryshaw + packages: ${{ steps.conda_build.outputs.packages }} + labels: ${{ needs.conda-update.outputs.DARC_CONDA_LABEL }} + force: true - name: Upload package to GitHub Release if: ${{ matrix.os == 'ubuntu-latest' }} - uses: ncipollo/release-action@v1.12.0 + uses: ncipollo/release-action@v1.21.0 with: allowUpdates: true - artifacts: | - /tmp/compilation-*/**/*.tar.bz2 + artifacts: ${{ steps.conda_build.outputs.packages_csv }} tag: "v${{ needs.conda-update.outputs.DARC_VERSION }}" token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 0be52c66..1468f3ec 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -18,13 +18,13 @@ jobs: runs-on: macos-latest steps: - name: Checkout đŸ›Žī¸ - uses: actions/checkout@v4 # If you're using actions/checkout@v4 you must set persist-credentials to false in most cases for the deployment to work correctly. + uses: actions/checkout@v7 with: persist-credentials: false - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v7 with: - python-version: '3.12' + python-version: '3.14' - name: Install and Build 🔧 # This example project is built using npm and outputs the result to the 'build' folder. Replace with the commands required to build your project, or remove this step entirely if your site is pre-built. run: | diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 4387ba9a..ddbe0380 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -12,12 +12,16 @@ on: branches: - '**' +permissions: + contents: read + packages: write + jobs: docker: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Prepare id: prep @@ -44,38 +48,33 @@ jobs: elif [ "${{ github.event_name }}" = "push" ]; then TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" fi - echo ::set-output name=version::${VERSION} - echo ::set-output name=tags::${TAGS} - echo ::set-output name=debug_tags::$(echo $TAGS | sed 's/darc/darc-debug/g') - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + { + echo "version=${VERSION}" + echo "tags=${TAGS}" + echo "debug_tags=$(echo "$TAGS" | sed 's/darc/darc-debug/g')" + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + } >> "$GITHUB_OUTPUT" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 # - name: Login to DockerHub # if: github.event_name != 'pull_request' - # uses: docker/login-action@v1 + # uses: docker/login-action@v3 # with: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.CR_PAT }} # ${{ secrets.GITHUB_TOKEN }} - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + password: ${{ secrets.GITHUB_TOKEN }} - name: Clean cache id: docker_prune @@ -84,7 +83,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . file: ./docker/Dockerfile @@ -106,7 +105,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Prepare id: prep @@ -133,38 +132,33 @@ jobs: elif [ "${{ github.event_name }}" = "push" ]; then TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" fi - echo ::set-output name=version::${VERSION} - echo ::set-output name=tags::${TAGS} - echo ::set-output name=debug_tags::$(echo $TAGS | sed 's/darc/darc-debug/g') - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + { + echo "version=${VERSION}" + echo "tags=${TAGS}" + echo "debug_tags=$(echo "$TAGS" | sed 's/darc/darc-debug/g')" + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + } >> "$GITHUB_OUTPUT" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 # - name: Login to DockerHub # if: github.event_name != 'pull_request' - # uses: docker/login-action@v1 + # uses: docker/login-action@v3 # with: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.CR_PAT }} # ${{ secrets.GITHUB_TOKEN }} - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + password: ${{ secrets.GITHUB_TOKEN }} - name: Clean cache id: docker_prune @@ -173,7 +167,7 @@ jobs: - name: Build and push id: docker_build_debug - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . file: ./docker/debug.dockerfile diff --git a/Pipfile b/Pipfile index 6cd27a7a..11cec50d 100644 --- a/Pipfile +++ b/Pipfile @@ -5,6 +5,9 @@ verify_ssl = true [dev-packages] python-darc = {editable = true,path = ".",extras = ["mysql"]} +build = "*" +setuptools = "*" +wheel = "*" autopep8 = "*" bandit = "*" mypy = "*" @@ -38,7 +41,7 @@ stem = "*" typing-extensions = "*" [requires] -python_version = "3" +python_version = "3.14" [pipenv] allow_prereleases = true diff --git a/conda/requirements.txt b/conda/requirements.txt index 86ff42e0..fa48061a 100644 --- a/conda/requirements.txt +++ b/conda/requirements.txt @@ -1,13 +1,13 @@ -beautifulsoup4 == 4.12.3 -file-magic == 0.4.1 -peewee == 3.17.5 -pottery == 3.0.0 -psutil == 6.0.0 -pymysql == 1.1.1 -python-datauri == 2.2.0 -redis == 4.6.0 -requests == 2.32.3 -requests-futures == 1.0.1 -selenium == 3.141.0 -stem == 1.8.2 -typing-extensions == 4.12.2 +beautifulsoup4==4.15.0 +file-magic==0.4.1 +peewee==4.3.0 +pottery==3.0.1 +psutil==7.2.2 +pymysql==1.2.0 +python-datauri==3.0.2 +redis==8.1.0 +requests==2.34.2 +requests-futures==1.0.2 +selenium==3.141.0 +stem==1.8.2 +typing-extensions==4.16.0 diff --git a/darc/__init__.py b/darc/__init__.py index c4bdb028..a254a4ae 100644 --- a/darc/__init__.py +++ b/darc/__init__.py @@ -52,4 +52,4 @@ from darc.sites import register as register_sites # pylint: disable=unused-import # noqa: F401 __all__ = ['darc'] -__version__ = '1.0.3' +__version__ = '1.0.4' diff --git a/docker/Dockerfile b/docker/Dockerfile index 487fef70..9818f68f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,160 +1,103 @@ -FROM ubuntu:focal +FROM python:3.14-bookworm LABEL org.opencontainers.image.title="darc" \ org.opencontainers.image.description="Darkweb Crawler Project" \ org.opencontainers.image.url="https://darc.jarryshaw.me/" \ org.opencontainers.image.source="https://github.com/JarryShaw/darc" \ - org.opencontainers.image.version="1.0.3" \ + org.opencontainers.image.version="1.0.4" \ org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License' STOPSIGNAL SIGINT HEALTHCHECK --interval=1h --timeout=1m \ - CMD wget https://httpbin.org/get -O /dev/null || exit 1 + CMD wget -q https://httpbin.org/get -O /dev/null || exit 1 ARG DARC_USER="darc" ENV LANG="C.UTF-8" \ LC_ALL="C.UTF-8" \ PYTHONIOENCODING="UTF-8" \ - DEBIAN_FRONTEND="teletype" \ + DEBIAN_FRONTEND="noninteractive" \ DARC_USER="${DARC_USER}" - # DEBIAN_FRONTEND="noninteractive" COPY extra/retry.sh /usr/local/bin/retry COPY extra/install.py /usr/local/bin/pty-install -#COPY vendor/jdk-11.0.18_linux-x64_bin.tar.gz /var/cache/oracle-jdk11-installer-local/ -RUN set -x \ - && apt-get update \ - && apt-get install --yes --no-install-recommends \ - apt-utils \ - && apt-get install --yes --no-install-recommends \ - gcc \ - g++ \ +# I2P's official repository supports Debian Bookworm but is not enabled by default. +RUN set -eux; \ + apt-get update; \ + apt-get install --yes --no-install-recommends \ + build-essential \ + ca-certificates \ + chromium \ + chromium-driver \ + curl \ + default-jre-headless \ + gnupg \ libmagic1 \ - make \ - software-properties-common \ + lsb-release \ + sudo \ tar \ + tor \ unzip \ wget \ - zlib1g-dev \ - && add-apt-repository ppa:deadsnakes/ppa --yes \ - && add-apt-repository ppa:linuxuprising/java --yes \ - && add-apt-repository ppa:i2p-maintainers/i2p --yes -RUN apt-get update \ - && apt-get install --yes --no-install-recommends \ - python3.9-dev \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - && ln -sf /usr/bin/python3.9 /usr/local/bin/python3 -# workaround for Git LFS as Docker Hub not supporting yet -RUN mkdir -p /var/cache/oracle-jdk11-installer-local/ \ - #&& version=$(apt-cache show oracle-java11-installer-local | grep Version | cut -d' ' -f2 | cut -d'-' -f1) \ - && wget -O /var/cache/oracle-jdk11-installer-local/jdk-11.0.18_linux-x64_bin.tar.gz \ - https://github.com/JarryShaw/darc/raw/master/vendor/jdk-11.0.18_linux-x64_bin.tar.gz -RUN pty-install --stdin '6\n70' apt-get install --yes --no-install-recommends \ - tzdata \ - && pty-install --stdin 'yes' apt-get install --yes \ - oracle-java11-installer-local -RUN apt-get install --yes --no-install-recommends \ - sudo \ - && adduser --disabled-password --gecos '' ${DARC_USER} \ - && adduser ${DARC_USER} sudo \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + zlib1g-dev; \ + curl --fail --location --silent --show-error \ + https://i2p.net/i2p-archive-keyring.gpg \ + --output /usr/share/keyrings/i2p-archive-keyring.gpg; \ + test "$(gpg --show-keys --with-colons /usr/share/keyrings/i2p-archive-keyring.gpg \ + | awk -F: '$1 == "fpr" { print $10; exit }')" = \ + "7840E7610F28B904753549D767ECE5605BCF1346"; \ + echo "deb [signed-by=/usr/share/keyrings/i2p-archive-keyring.gpg] https://deb.i2p.net/ bookworm main" \ + > /etc/apt/sources.list.d/i2p.list; \ + apt-get update; \ + apt-get install --yes --no-install-recommends i2p i2p-keyring; \ + ln -s /usr/bin/chromium /usr/local/bin/google-chrome; \ + adduser --disabled-password --gecos '' "${DARC_USER}"; \ + adduser "${DARC_USER}" sudo; \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -## Tor -RUN apt-get install --yes --no-install-recommends tor COPY extra/torrc.focal /etc/tor/torrc - -## I2P -RUN apt-get install --yes --no-install-recommends i2p -COPY extra/i2p.focal /etc/defaults/i2p +COPY extra/i2p.focal /etc/default/i2p ## ZeroNet -COPY vendor/ZeroNet-linux-dist-linux64.tar.gz /tmp -RUN set -x \ - && cd /tmp \ - && tar xvpfz ZeroNet-linux-dist-linux64.tar.gz \ - && mv ZeroNet-linux-dist-linux64 /usr/local/src/zeronet +COPY vendor/ZeroNet-linux-dist-linux64.tar.gz /tmp/ +RUN set -eux; \ + tar xzf /tmp/ZeroNet-linux-dist-linux64.tar.gz -C /tmp; \ + mv /tmp/ZeroNet-linux-dist-linux64 /usr/local/src/zeronet COPY extra/zeronet.focal.conf /usr/local/src/zeronet/zeronet.conf ## FreeNet +COPY vendor/new_installer_offline.jar /tmp/ USER darc -COPY vendor/new_installer_offline.jar /tmp -RUN set -x \ - && cd /tmp \ - && ( pty-install --stdin '/home/darc/freenet\n1' java -jar new_installer_offline.jar || true ) \ - && sudo mv /home/darc/freenet /usr/local/src/freenet +RUN set -eux; \ + (pty-install --stdin '/home/darc/freenet\n1' java -jar /tmp/new_installer_offline.jar || true); \ + sudo mv /home/darc/freenet /usr/local/src/freenet USER root ## NoIP -COPY vendor/noip-duc-linux.tar.gz /tmp -RUN set -x \ - && cd /tmp \ - && tar xvpfz noip-duc-linux.tar.gz \ - && mv noip-2.1.9-1 /usr/local/src/noip \ - && cd /usr/local/src/noip \ - && make - # && make install - -# # set up timezone -# RUN echo 'Asia/Shanghai' > /etc/timezone \ -# && rm -f /etc/localtime \ -# && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ -# && dpkg-reconfigure -f noninteractive tzdata - -COPY vendor/chromedriver_linux64.zip /tmp/ - #vendor/google-chrome-stable_current_amd64.deb /tmp/ -# workaround for Git LFS as Docker Hub not supporting yet -RUN wget -O /tmp/google-chrome-stable_current_amd64.deb \ - https://github.com/JarryShaw/darc/raw/master/vendor/google-chrome-stable_current_amd64.deb -RUN set -x \ - ## ChromeDriver - && unzip -d /usr/bin /tmp/chromedriver_linux64.zip \ - && which chromedriver \ - ## Google Chrome - && ( dpkg --install /tmp/google-chrome-stable_current_amd64.deb || true ) \ - && apt-get install --fix-broken --yes --no-install-recommends \ - && dpkg --install /tmp/google-chrome-stable_current_amd64.deb \ - && which google-chrome - -# Using pip: -COPY requirements.txt /tmp -RUN python3 -m pip install -r /tmp/requirements.txt --no-cache-dir - -RUN set -x \ - && rm -rf \ - ## APT repository lists +COPY vendor/noip-duc-linux.tar.gz /tmp/ +RUN set -eux; \ + tar xzf /tmp/noip-duc-linux.tar.gz -C /tmp; \ + mv /tmp/noip-2.1.9-1 /usr/local/src/noip; \ + make -C /usr/local/src/noip + +COPY requirements.txt /tmp/requirements.txt +RUN set -eux; \ + python -m pip install --no-cache-dir --upgrade pip setuptools wheel; \ + python -m pip install --no-cache-dir -r /tmp/requirements.txt; \ + rm -rf \ /var/lib/apt/lists/* \ - ## Python dependencies - /tmp/requirements.txt \ - /tmp/pip \ - ## ChromeDriver - /tmp/chromedriver_linux64.zip \ - ## Google Chrome - /tmp/google-chrome-stable_current_amd64.deb \ - ## Vendors + /root/.cache \ /tmp/new_installer_offline.jar \ /tmp/noip-duc-linux.tar.gz \ - /tmp/ZeroNet-linux-dist-linux64.tar.gz \ - #&& apt-get remove --auto-remove --yes \ - # software-properties-common \ - # unzip \ - && apt-get autoremove -y \ - && apt-get autoclean \ - && apt-get clean - -ENTRYPOINT [ "python3", "-m", "darc" ] -#ENTRYPOINT [ "bash", "/app/run.sh" ] -CMD [ "--help" ] + /tmp/requirements.txt \ + /tmp/ZeroNet-linux-dist-linux64.tar.gz; \ + apt-get autoremove --yes; \ + apt-get clean WORKDIR /app COPY darc/ /app/darc/ -COPY LICENSE \ - MANIFEST.in \ - README.rst \ - extra/run.sh \ - setup.cfg \ - setup.py \ - test_darc.py /app/ -RUN python3 -m pip install -e . +COPY LICENSE MANIFEST.in README.rst extra/run.sh setup.cfg setup.py test_darc.py /app/ +RUN python -m pip install --no-cache-dir --no-deps -e . + +ENTRYPOINT [ "python", "-m", "darc" ] +CMD [ "--help" ] diff --git a/docker/debug.dockerfile b/docker/debug.dockerfile index b6b85fef..99be9f06 100644 --- a/docker/debug.dockerfile +++ b/docker/debug.dockerfile @@ -1,171 +1,90 @@ -# Python support can be specified down to the minor or micro version -# (e.g. 3.6 or 3.6.3). -# OS Support also exists for jessie & stretch (slim and full). -# See https://hub.docker.com/r/library/python/ for all supported Python -# tags from Docker Hub. -#FROM python:3.9-alpine -#FROM python:3.9 - -# If you prefer miniconda: -#FROM continuumio/miniconda3 - -FROM ubuntu:focal +FROM python:3.14-bookworm LABEL org.opencontainers.image.title="darc" \ org.opencontainers.image.description="Darkweb Crawler Project" \ org.opencontainers.image.url="https://darc.jarryshaw.me/" \ org.opencontainers.image.source="https://github.com/JarryShaw/darc" \ - org.opencontainers.image.version="1.0.3" \ + org.opencontainers.image.version="1.0.4" \ org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License' -#EXPOSE 9050 STOPSIGNAL SIGINT HEALTHCHECK --interval=1h --timeout=1m \ - CMD wget https://httpbin.org/get -O /dev/null || exit 1 + CMD wget -q https://httpbin.org/get -O /dev/null || exit 1 ARG DARC_USER="darc" ENV LANG="C.UTF-8" \ LC_ALL="C.UTF-8" \ PYTHONIOENCODING="UTF-8" \ - DEBIAN_FRONTEND="teletype" \ + DEBIAN_FRONTEND="noninteractive" \ DARC_USER="${DARC_USER}" - # DEBIAN_FRONTEND="noninteractive" - -# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories \ -# && apk add --update --no-cache \ -# chromium \ -# chromium-chromedriver \ -# tor -# COPY extra/torrc.alpine /etc/tor/torrc COPY extra/retry.sh /usr/local/bin/retry COPY extra/install.py /usr/local/bin/pty-install -#COPY vendor/jdk-11.0.18_linux-x64_bin.tar.gz /var/cache/oracle-jdk11-installer-local/ -RUN set -x \ - && apt-get update \ - && apt-get install --yes \ - apt-utils \ - && apt-get install --yes \ - apt-transport-https \ - ca-certificates -COPY extra/sources.focal.list /etc/apt/sources.list -RUN set -x \ - && apt-get update \ - && apt-get install --yes \ - gcc \ - g++ \ +RUN set -eux; \ + apt-get update; \ + apt-get install --yes --no-install-recommends \ + build-essential \ + ca-certificates \ + chromium \ + chromium-driver \ + curl \ + default-jre-headless \ + gnupg \ libmagic1 \ - make \ - software-properties-common \ + lsb-release \ + sudo \ tar \ + tor \ unzip \ wget \ - zlib1g-dev \ - && add-apt-repository ppa:deadsnakes/ppa --yes \ - && add-apt-repository ppa:linuxuprising/java --yes \ - && add-apt-repository ppa:i2p-maintainers/i2p --yes -RUN apt-get update \ - && apt-get install --yes \ - python3.9-dev \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - && ln -sf /usr/bin/python3.9 /usr/local/bin/python3 -# workaround for Git LFS as Docker Hub not supporting yet -RUN mkdir -p /var/cache/oracle-jdk11-installer-local/ \ - #&& version=$(apt-cache show oracle-java11-installer-local | grep Version | cut -d' ' -f2 | cut -d'-' -f1) \ - && wget -O /var/cache/oracle-jdk11-installer-local/jdk-11.0.18_linux-x64_bin.tar.gz \ - https://github.com/JarryShaw/darc/raw/master/vendor/jdk-11.0.18_linux-x64_bin.tar.gz -RUN pty-install --stdin '6\n70' apt-get install --yes --no-install-recommends \ - tzdata \ - && pty-install --stdin 'yes' apt-get install --yes \ - oracle-java11-installer-local -RUN apt-get install --yes \ - sudo \ - && adduser --disabled-password --gecos '' ${DARC_USER} \ - && adduser ${DARC_USER} sudo \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + zlib1g-dev; \ + curl --fail --location --silent --show-error \ + https://i2p.net/i2p-archive-keyring.gpg \ + --output /usr/share/keyrings/i2p-archive-keyring.gpg; \ + test "$(gpg --show-keys --with-colons /usr/share/keyrings/i2p-archive-keyring.gpg \ + | awk -F: '$1 == "fpr" { print $10; exit }')" = \ + "7840E7610F28B904753549D767ECE5605BCF1346"; \ + echo "deb [signed-by=/usr/share/keyrings/i2p-archive-keyring.gpg] https://deb.i2p.net/ bookworm main" \ + > /etc/apt/sources.list.d/i2p.list; \ + apt-get update; \ + apt-get install --yes --no-install-recommends i2p i2p-keyring; \ + ln -s /usr/bin/chromium /usr/local/bin/google-chrome; \ + adduser --disabled-password --gecos '' "${DARC_USER}"; \ + adduser "${DARC_USER}" sudo; \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -## Tor -RUN apt-get install --yes tor COPY extra/torrc.focal /etc/tor/torrc +COPY extra/i2p.focal /etc/default/i2p -## I2P -RUN apt-get install --yes i2p -COPY extra/i2p.focal /etc/defaults/i2p - -## ZeroNet -COPY vendor/ZeroNet-linux-dist-linux64.tar.gz /tmp -RUN set -x \ - && cd /tmp \ - && tar xvpfz ZeroNet-linux-dist-linux64.tar.gz \ - && mv ZeroNet-linux-dist-linux64 /usr/local/src/zeronet +COPY vendor/ZeroNet-linux-dist-linux64.tar.gz /tmp/ +RUN set -eux; \ + tar xzf /tmp/ZeroNet-linux-dist-linux64.tar.gz -C /tmp; \ + mv /tmp/ZeroNet-linux-dist-linux64 /usr/local/src/zeronet COPY extra/zeronet.focal.conf /usr/local/src/zeronet/zeronet.conf -## FreeNet +COPY vendor/new_installer_offline.jar /tmp/ USER darc -COPY vendor/new_installer_offline.jar /tmp -RUN set -x \ - && cd /tmp \ - && ( pty-install --stdin '/home/darc/freenet\n1' java -jar new_installer_offline.jar || true ) \ - && sudo mv /home/darc/freenet /usr/local/src/freenet +RUN set -eux; \ + (pty-install --stdin '/home/darc/freenet\n1' java -jar /tmp/new_installer_offline.jar || true); \ + sudo mv /home/darc/freenet /usr/local/src/freenet USER root -## NoIP -COPY vendor/noip-duc-linux.tar.gz /tmp -RUN set -x \ - && cd /tmp \ - && tar xf noip-duc-linux.tar.gz \ - && mv noip-2.1.9-1 /usr/local/src/noip \ - && cd /usr/local/src/noip \ - && make - # && make install - -# # set up timezone -# RUN echo 'Asia/Shanghai' > /etc/timezone \ -# && rm -f /etc/localtime \ -# && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ -# && dpkg-reconfigure -f noninteractive tzdata +COPY vendor/noip-duc-linux.tar.gz /tmp/ +RUN set -eux; \ + tar xzf /tmp/noip-duc-linux.tar.gz -C /tmp; \ + mv /tmp/noip-2.1.9-1 /usr/local/src/noip; \ + make -C /usr/local/src/noip -# ADD tbb/tor-browser-linux64-8.5.5_en-US.tar.gz / -# ADD driver/geckodriver-v0.26.0-linux64.tar.gz /usr/local/bin -COPY vendor/chromedriver_linux64.zip /tmp/ - #vendor/google-chrome-stable_current_amd64.deb /tmp/ -# workaround for Git LFS as Docker Hub not supporting yet -RUN wget -O /tmp/google-chrome-stable_current_amd64.deb \ - https://github.com/JarryShaw/darc/raw/master/vendor/google-chrome-stable_current_amd64.deb -RUN set -x \ - ## ChromeDriver - && unzip -d /usr/bin /tmp/chromedriver_linux64.zip \ - && which chromedriver \ - ## Google Chrome - && (dpkg --install /tmp/google-chrome-stable_current_amd64.deb || true) \ - && apt-get install --fix-broken --yes \ - && dpkg --install /tmp/google-chrome-stable_current_amd64.deb \ - && which google-chrome - -# Using pip: -COPY requirements.debug.txt /tmp -RUN set -x \ - && python3 -m pip install -r /tmp/requirements.debug.txt --cache-dir /app/cache \ - && python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ - && python3 -m pip install ipython -#CMD ["python3", "-m", "darc"] - -ENTRYPOINT [ "python3", "-m", "darc" ] -#ENTRYPOINT [ "bash", "/app/extra/run.sh" ] -CMD [ "--help" ] - -# Using pipenv: -#RUN python3 -m pip install pipenv -#RUN pipenv install --ignore-pipfile -#CMD ["pipenv", "run", "python3", "-m", "darc"] - -# Using miniconda (make sure to replace 'myenv' w/ your environment name): -#RUN conda env create -f environment.yml -#CMD /bin/bash -c "source activate myenv && python3 -m darc" +COPY requirements.txt /tmp/requirements.txt +RUN set -eux; \ + python -m pip install --upgrade pip setuptools wheel; \ + python -m pip install --cache-dir /app/cache -r /tmp/requirements.txt; \ + python -m pip install ipython WORKDIR /app ADD . /app -RUN python3 -m pip install -e . +RUN python -m pip install --no-deps -e . + +ENTRYPOINT [ "python", "-m", "darc" ] +CMD [ "--help" ] diff --git a/docs/source/conf.py b/docs/source/conf.py index 9268ecb5..8f8f277e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,7 +37,7 @@ author = 'Jarry Shaw' # The full version, including alpha/beta/rc tags -release = '1.0.3' +release = '1.0.4' # -- General configuration --------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index ac20e304..288b214c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,50 +1,29 @@ -# Python packages -pip==24.0 -setuptools==69.1.1 -wheel==0.42.0 - -# Python dependencies -# -# pipenv lock --requirements -# - --i https://pypi.tuna.tsinghua.edu.cn/simple ---extra-index-url https://mirrors.sjtug.sjtu.edu.cn/pypi/web/simple ---extra-index-url https://pypi.org/simple -async-generator==1.10; python_version >= '3.5' -attrs==23.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -beautifulsoup4[html5lib]==4.12.3 -certifi==2024.2.2 -cffi==1.16.0 -chardet==5.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -cryptography==42.0.5 -file-magic==0.4.1 -h11==0.14.0; python_version >= '3.6' -hiredis==2.3.2 -html5lib==1.1 -idna==3.7; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -mmh3==4.1.0 -outcome==1.3.0.post0; python_version >= '3.6' -peewee==3.17.1 -pottery==3.0.0 -psutil==5.9.8 -pycparser==2.21; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -pymysql[rsa]==1.1.0 -pyopenssl==24.1.0 -pysocks==1.7.1 -python-datauri==2.1.0 -redis[hiredis]==5.0.3 -requests-futures==1.0.1 -requests[socks]==2.31.0 +-i https://pypi.org/simple +beautifulsoup4[html5lib]==4.15.0; python_full_version >= '3.7.0' +cached-property==2.0.1; python_version >= '3.8' +certifi==2026.7.22; python_version >= '3.7' +cffi==2.1.1; python_version >= '3.10' +charset-normalizer==3.5.1; python_version >= '3.7' +cryptography==50.0.0; python_version >= '3.9' and python_full_version not in '3.9.0, 3.9.1' +file-magic==0.4.1; python_version >= '2.7' +hiredis==3.4.1; python_version >= '3.8' +html5lib==1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +idna==3.19; python_version >= '3.9' +mmh3==5.2.1; python_version >= '3.10' +peewee==4.3.0; python_version >= '3.8' +pottery==3.0.1; python_version >= '3.9' and python_version < '4' +psutil==7.2.2; python_version >= '3.6' +pycparser==3.0; python_version >= '3.10' +pymysql[rsa]==1.2.0; python_version >= '3.9' +pysocks==1.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +python-datauri==3.0.2 +redis[hiredis]==8.1.0; python_version >= '3.10' +requests[socks]==2.34.2; python_version >= '3.10' +requests-futures==1.0.2 selenium==3.141.0 -six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -sniffio==1.3.1; python_version >= '3.5' -sortedcontainers==2.4.0 -soupsieve==2.5; python_version >= '3.0' +six==1.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' +soupsieve==2.9.2; python_version >= '3.10' stem==1.8.2 -trio-websocket==0.11.1; python_version >= '3.5' -trio==0.24.0; python_version >= '3.6' -typing-extensions==4.10.0 -urllib3[secure]==2.2.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' -webencodings==0.5.1 -wsproto==1.2.0; python_full_version >= '3.6.1' +typing-extensions==4.16.0; python_version >= '3.9' +urllib3==2.7.0; python_version >= '3.10' +webencodings==0.6.1; python_version >= '3.10' diff --git a/setup.py b/setup.py index 2ec08d75..f6ea3309 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ version_info = sys.version_info[:2] # version string -__version__ = '1.0.3' +__version__ = '1.0.4' # setup attributes attrs = dict( diff --git a/util/conda-dist.py b/util/conda-dist.py index e0666ef6..a30beeab 100644 --- a/util/conda-dist.py +++ b/util/conda-dist.py @@ -2,24 +2,42 @@ import importlib.metadata as imp_meta import os.path as os_path +import re import packaging.requirements as pkg_req req_file = os_path.join('conda', 'requirements.txt') +spec_re = re.compile(r'\s*(===|!=|~=|==|<=|>=|<|>)\s*') + + +def normalise_requirement(line): + req = line.strip() + if not req or req.startswith('#'): + return None + + if ';' in req: + spec, marker = req.split(';', 1) + return f'{spec_re.sub(r"\1", spec.strip())}; {marker.strip()}' + return spec_re.sub(r'\1', req) + + +def pinned_requirement(req, version): + spec = f'{req.name}=={version}' + if req.marker is None: + return spec + return f'{spec}; {req.marker}' data = {} # dict[str, str] with open(req_file) as file: for line in file: - if not line.strip(): + line = normalise_requirement(line) + if line is None: continue req = pkg_req.Requirement(line) try: ver = imp_meta.version(req.name) - if req.marker is None: - data[req.name] = f'{req.name} == {ver}' - else: - data[req.name] = f'{req.name} == {ver} ; {req.marker}' + data[req.name] = pinned_requirement(req, ver) except imp_meta.PackageNotFoundError: if req.marker is None: raise From 9226aa8d88818b4fe09af7b628781b0cd9b3811b Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 21 Aug 2026 16:56:34 -0400 Subject: [PATCH 2/3] Limit container publishing to release refs --- .github/workflows/docker-image.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index ddbe0380..8edce404 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -69,7 +69,11 @@ jobs: # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - if: github.event_name != 'pull_request' + if: >- + github.event_name == 'schedule' || + (github.event_name == 'push' && + (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || + startsWith(github.ref, 'refs/tags/'))) uses: docker/login-action@v3 with: registry: ghcr.io @@ -88,7 +92,11 @@ jobs: context: . file: ./docker/Dockerfile platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} + push: >- + ${{ github.event_name == 'schedule' || + (github.event_name == 'push' && + (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || + startsWith(github.ref, 'refs/tags/'))) }} tags: ${{ steps.prep.outputs.tags }} labels: | org.opencontainers.image.title=${{ github.event.repository.name }} @@ -153,7 +161,11 @@ jobs: # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - if: github.event_name != 'pull_request' + if: >- + github.event_name == 'schedule' || + (github.event_name == 'push' && + (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || + startsWith(github.ref, 'refs/tags/'))) uses: docker/login-action@v3 with: registry: ghcr.io @@ -172,7 +184,11 @@ jobs: context: . file: ./docker/debug.dockerfile platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} + push: >- + ${{ github.event_name == 'schedule' || + (github.event_name == 'push' && + (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || + startsWith(github.ref, 'refs/tags/'))) }} tags: ${{ steps.prep.outputs.debug_tags }} labels: | org.opencontainers.image.title=${{ github.event.repository.name }} From 517cfbceeb5aa58e5414ce06a6e5876238a65ad2 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 21 Aug 2026 17:00:46 -0400 Subject: [PATCH 3/3] Add Python compatibility CI --- .github/workflows/docker-image.yml | 20 ++++----- .github/workflows/python-compatibility.yml | 52 ++++++++++++++++++++++ setup.py | 11 ++--- 3 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/python-compatibility.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 8edce404..63371bfb 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -56,14 +56,14 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4.3.0 # - name: Login to DockerHub # if: github.event_name != 'pull_request' - # uses: docker/login-action@v3 + # uses: docker/login-action@v4.6.0 # with: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -74,7 +74,7 @@ jobs: (github.event_name == 'push' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) - uses: docker/login-action@v3 + uses: docker/login-action@v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -87,7 +87,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7.3.0 with: context: . file: ./docker/Dockerfile @@ -148,14 +148,14 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4.3.0 # - name: Login to DockerHub # if: github.event_name != 'pull_request' - # uses: docker/login-action@v3 + # uses: docker/login-action@v4.6.0 # with: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -166,7 +166,7 @@ jobs: (github.event_name == 'push' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) - uses: docker/login-action@v3 + uses: docker/login-action@v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -179,7 +179,7 @@ jobs: - name: Build and push id: docker_build_debug - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7.3.0 with: context: . file: ./docker/debug.dockerfile diff --git a/.github/workflows/python-compatibility.yml b/.github/workflows/python-compatibility.yml new file mode 100644 index 00000000..2df4738b --- /dev/null +++ b/.github/workflows/python-compatibility.yml @@ -0,0 +1,52 @@ +name: "Python Compatibility" + +on: + push: + branches: [master] + pull_request: + branches: [master] + schedule: + - cron: '0 4 * * 6' + +permissions: + contents: read + +jobs: + compatibility: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental == true }} + strategy: + fail-fast: false + matrix: + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + experimental: + - false + include: + - python-version: "3.15" + experimental: true + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: ${{ matrix.experimental }} + + - name: Install package + run: | + python -m pip install -U pip setuptools wheel + python -m pip install -e . + + - name: Verify package + env: + DARC_USER: nobody + TOR_PASS: 'null' + run: | + python -m compileall -q darc + python -c 'import darc; print(darc.__version__)' diff --git a/setup.py b/setup.py index f6ea3309..25991923 100644 --- a/setup.py +++ b/setup.py @@ -58,9 +58,11 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Programming Language :: Python :: 3 :: Only', 'Topic :: Software Development', 'Topic :: Utilities', @@ -103,7 +105,6 @@ # password # fullname # long_description_content_type - # python_requires # zip_safe, install_requires=[ 'beautifulsoup4[html5lib]', @@ -162,7 +163,7 @@ # password # fullname long_description_content_type='text/x-rst', - python_requires='>=3.6', + python_requires='>=3.10', # zip_safe=True, )) except ImportError: