From 48e8fc47ae190969e603fd701c36a8045e6f5578 Mon Sep 17 00:00:00 2001 From: tishin-endou Date: Thu, 28 May 2026 22:39:12 +0800 Subject: [PATCH 1/4] Add AWS S3 SigV4 region CI tests Co-Authored-By: An Qiuyu --- .github/scripts/generate_ci_config.sh | 54 ++++++++++++++++++++++++++- .github/workflows/e2e-test.yml | 23 ++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index fa1fc7f..424caf0 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -3,7 +3,7 @@ set -xeuo pipefail if [[ $# -lt 2 ]]; then cat >&2 <<'USAGE' -Usage: generate_ci_config.sh [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] +Usage: generate_ci_config.sh [--minio] [--aws-s3] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] USAGE exit 1 fi @@ -12,6 +12,7 @@ OUTPUT=$1 BASE_CONFIG=$2 shift 2 MINIO=false +AWS_S3=false JUPYTERHUB=false WEKO=false FLOWABLE=false @@ -23,6 +24,9 @@ for arg in "$@"; do --minio) MINIO=true ;; + --aws-s3) + AWS_S3=true + ;; --jupyterhub) JUPYTERHUB=true ;; @@ -46,6 +50,11 @@ for arg in "$@"; do done +if [[ "${MINIO}" == "true" && "${AWS_S3}" == "true" ]]; then + echo "--minio and --aws-s3 both write storages_s3 and cannot be used together" >&2 + exit 1 +fi + cp "${BASE_CONFIG}" "${OUTPUT}" if [[ "${MINIO}" == "true" ]]; then @@ -74,13 +83,54 @@ s3compat_test_bucket_name_2: '${S3COMPAT_BUCKET_NAME_2}' s3compat_type_name_1: '${S3COMPAT_SERVICE_NAME}' s3compat_type_name_2: '${S3COMPAT_SERVICE_NAME}' EOF -else +elif [[ "${AWS_S3}" != "true" ]]; then cat >> "${OUTPUT}" <<'EOF' storages_s3: [] EOF fi +if [[ "${AWS_S3}" == "true" ]]; then + required_aws_s3_vars=( + AWS_S3_ACCESS_KEY + AWS_S3_SECRET_KEY + AWS_S3_LEGACY_REGION + AWS_S3_LEGACY_BUCKET_NAME + AWS_S3_V4_REGION + AWS_S3_V4_BUCKET_NAME + ) + + missing_aws_s3_vars=() + for var_name in "${required_aws_s3_vars[@]}"; do + if [[ -z "${!var_name:-}" ]]; then + missing_aws_s3_vars+=("${var_name}") + fi + done + + if [[ ${#missing_aws_s3_vars[@]} -gt 0 ]]; then + echo "AWS S3 test credentials are not set: ${missing_aws_s3_vars[*]}" >&2 + exit 1 + fi + + cat >> "${OUTPUT}" <&2 diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 5dfa497..c34d693 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -162,6 +162,19 @@ jobs: weko_enabled: false jupyterhub_enabled: false flowable_enabled: false + - name: user-aws-s3 + display_name: "User Tests (AWS S3 SigV4 Regions)" + include_admin: false + skip_admin: true + skip_metadata: true + skip_default_storage: true + skip_login: true + skip_130mb_upload: true + minio_enabled: false + aws_s3_enabled: true + weko_enabled: false + jupyterhub_enabled: false + flowable_enabled: false - name: admin-minio display_name: "Admin Tests (MinIO)" include_admin: true @@ -215,6 +228,7 @@ jobs: name: E2E ${{ matrix.test-group.display_name }} env: MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }} + AWS_S3_ENABLED: ${{ matrix.test-group.aws_s3_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }} WEKO_ENABLED: ${{ matrix.test-group.weko_enabled == true && 'true' || 'false' }} @@ -226,6 +240,12 @@ jobs: MINIO_ENDPOINT: http://minio:9000 MINIO_REGION: us-east-1 MINIO_SERVICE_NAME: MinIO (CI) + AWS_S3_ACCESS_KEY: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY || '' }} + AWS_S3_SECRET_KEY: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY || '' }} + AWS_S3_LEGACY_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_REGION || '' }} + AWS_S3_LEGACY_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_BUCKET_NAME || '' }} + AWS_S3_V4_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_REGION || '' }} + AWS_S3_V4_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_BUCKET_NAME || '' }} steps: - name: Checkout test repository @@ -1078,6 +1098,9 @@ jobs: if [ "${MINIO_ENABLED}" = "true" ]; then args+=(--minio) fi + if [ "${AWS_S3_ENABLED}" = "true" ]; then + args+=(--aws-s3) + fi if [ "${JUPYTERHUB_ENABLED}" = "true" ]; then args+=(--jupyterhub) fi From 1b8e51f32a29426eadb32bd5f25a65b0e18c7939 Mon Sep 17 00:00:00 2001 From: tishin-endou Date: Sat, 13 Jun 2026 11:44:31 +0800 Subject: [PATCH 2/4] fix(ci): separate S3 credentials and mask artifacts --- .github/scripts/generate_ci_config.sh | 24 ++++-- .github/scripts/mask_sensitive_artifacts.sh | 91 +++++++++++++++++++++ .github/workflows/e2e-test.yml | 34 +++++++- 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/mask_sensitive_artifacts.sh diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index 424caf0..d31a4dd 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -92,8 +92,10 @@ fi if [[ "${AWS_S3}" == "true" ]]; then required_aws_s3_vars=( - AWS_S3_ACCESS_KEY - AWS_S3_SECRET_KEY + AWS_S3_ACCESS_KEY_1 + AWS_S3_SECRET_KEY_1 + AWS_S3_ACCESS_KEY_2 + AWS_S3_SECRET_KEY_2 AWS_S3_LEGACY_REGION AWS_S3_LEGACY_BUCKET_NAME AWS_S3_V4_REGION @@ -112,6 +114,16 @@ if [[ "${AWS_S3}" == "true" ]]; then exit 1 fi + if [[ "${AWS_S3_ACCESS_KEY_1}" == "${AWS_S3_ACCESS_KEY_2}" ]]; then + echo "AWS_S3_ACCESS_KEY_1 and AWS_S3_ACCESS_KEY_2 must be different" >&2 + exit 1 + fi + + if [[ "${AWS_S3_SECRET_KEY_1}" == "${AWS_S3_SECRET_KEY_2}" ]]; then + echo "AWS_S3_SECRET_KEY_1 and AWS_S3_SECRET_KEY_2 must be different" >&2 + exit 1 + fi + cat >> "${OUTPUT}" < [ ...]" >&2 + exit 1 +fi + +base_dir=$(pwd) + +mask_value() { + local value="$1" + local length=${#value} + if (( length <= 8 )); then + printf '****' + return + fi + + local prefix="${value:0:4}" + local suffix="${value: -4}" + local mask_length=$((length - 8)) + local stars + stars=$(printf '%*s' "$mask_length" '' | tr ' ' '*') + printf '%s%s%s' "$prefix" "$stars" "$suffix" +} + +replace_in_file() { + local file="$1" + local value="$2" + local masked="$3" + + VALUE="$value" MASKED="$masked" perl -0pi -e 's/\Q$ENV{VALUE}\E/$ENV{MASKED}/g' "$file" +} + +replace_in_tree() { + local target_dir="$1" + local value="$2" + local masked="$3" + + find "$target_dir" -type f \ + \( -name '*.ipynb' -o -name '*.log' -o -name '*.json' -o -name '*.har' -o -name '*.html' -o -name '*.js' -o -name '*.txt' -o -name '*.yaml' -o -name '*.yml' \) \ + -print0 | + while IFS= read -r -d '' file; do + replace_in_file "$file" "$value" "$masked" + done + + find "$target_dir" -type f -name 'har.zip' -print0 | + while IFS= read -r -d '' zip_file; do + local tmp_dir + local output_zip + tmp_dir=$(mktemp -d) + case "$zip_file" in + /*) + output_zip="$zip_file" + ;; + *) + output_zip="$base_dir/$zip_file" + ;; + esac + unzip -q "$zip_file" -d "$tmp_dir" + find "$tmp_dir" -type f -print0 | + while IFS= read -r -d '' file; do + replace_in_file "$file" "$value" "$masked" + done + (cd "$tmp_dir" && zip -qr "$output_zip" .) + rm -rf "$tmp_dir" + done +} + +while IFS='=' read -r name value; do + case "$name" in + *ACCESS_KEY*|*SECRET_KEY*|*PASSWORD*|*TOKEN*) + ;; + *) + continue + ;; + esac + + if [[ -z "$value" || ${#value} -lt 8 ]]; then + continue + fi + + masked=$(mask_value "$value") + echo "::add-mask::$value" + + for target_dir in "$@"; do + if [[ -d "$target_dir" ]]; then + replace_in_tree "$target_dir" "$value" "$masked" + fi + done +done < <(env) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c34d693..7f7031c 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -240,8 +240,10 @@ jobs: MINIO_ENDPOINT: http://minio:9000 MINIO_REGION: us-east-1 MINIO_SERVICE_NAME: MinIO (CI) - AWS_S3_ACCESS_KEY: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY || '' }} - AWS_S3_SECRET_KEY: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY || '' }} + AWS_S3_ACCESS_KEY_1: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY_1 || '' }} + AWS_S3_SECRET_KEY_1: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY_1 || '' }} + AWS_S3_ACCESS_KEY_2: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY_2 || '' }} + AWS_S3_SECRET_KEY_2: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY_2 || '' }} AWS_S3_LEGACY_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_REGION || '' }} AWS_S3_LEGACY_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_BUCKET_NAME || '' }} AWS_S3_V4_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_REGION || '' }} @@ -1042,6 +1044,28 @@ jobs: docker-compose exec -T web python3 -m scripts.register_erad_metadata /tmp/erad_sample.csv echo "e-Rad data registered successfully" + - name: Validate and mask sensitive test credentials + working-directory: e2e-tests + run: | + set -euo pipefail + + bash .github/scripts/mask_sensitive_artifacts.sh /tmp/nonexistent-mask-target + + if [ "${AWS_S3_ENABLED}" = "true" ]; then + if [ -z "${AWS_S3_ACCESS_KEY_1}" ] || [ -z "${AWS_S3_SECRET_KEY_1}" ] || [ -z "${AWS_S3_ACCESS_KEY_2}" ] || [ -z "${AWS_S3_SECRET_KEY_2}" ]; then + echo "AWS S3 test credentials are not fully configured. Set AWS_S3_ACCESS_KEY_1, AWS_S3_SECRET_KEY_1, AWS_S3_ACCESS_KEY_2, and AWS_S3_SECRET_KEY_2 secrets." + exit 1 + fi + if [ "${AWS_S3_ACCESS_KEY_1}" = "${AWS_S3_ACCESS_KEY_2}" ]; then + echo "AWS_S3_ACCESS_KEY_1 and AWS_S3_ACCESS_KEY_2 must be different." + exit 1 + fi + if [ "${AWS_S3_SECRET_KEY_1}" = "${AWS_S3_SECRET_KEY_2}" ]; then + echo "AWS_S3_SECRET_KEY_1 and AWS_S3_SECRET_KEY_2 must be different." + exit 1 + fi + fi + - name: Prepare test configuration working-directory: e2e-tests run: | @@ -1175,6 +1199,12 @@ jobs: echo "ticket=$TICKET" >> $GITHUB_OUTPUT echo "Extracted ticket: $TICKET" + - name: Mask sensitive values in test artifacts + if: always() + working-directory: e2e-tests + run: | + bash .github/scripts/mask_sensitive_artifacts.sh result result-failed + - name: Generate Excel summary if: always() working-directory: e2e-tests From 0e83912dcba8b0eaa9e45c339085574e104e0417 Mon Sep 17 00:00:00 2001 From: tishin-endou Date: Mon, 20 Jul 2026 09:55:47 +0900 Subject: [PATCH 3/4] =?UTF-8?q?feat(s3):=20Amazon=20S3=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AEE2E?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=83=8E=E3=83=BC=E3=83=88=E3=83=96?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - テスト手順-管理者機能-S3-機関ストレージ.ipynb を新規作成 (S3CompatSigV4版をベースに、Amazon S3用にエンドポイントURL関連を除去) URL/entityIDは実環境の固有ホスト名を残さず、他ノートブックと同様に example.comプレースホルダ or None(.config.yaml/プロンプト入力で補完)とした - 取りまとめ-S3共通.ipynb: .config.yaml から未設定パラメータを補完する ローダーセルを追加（対話実行を容易にするため） - .gitignore: .claude/ (ローカルのAI開発ツール設定) を除外 --- .gitignore | 3 + ...343\203\254\343\203\274\343\202\270.ipynb" | 1171 +++++++++++++++++ ...\202\201-S3\345\205\261\351\200\232.ipynb" | 25 + 3 files changed, 1199 insertions(+) create mode 100644 "\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\347\256\241\347\220\206\350\200\205\346\251\237\350\203\275-S3-\346\251\237\351\226\242\343\202\271\343\203\210\343\203\254\343\203\274\343\202\270.ipynb" diff --git a/.gitignore b/.gitignore index c524767..65b652f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ venv/ *~ .DS_Store +# Claude Code / Claude in Cowork local settings (machine-specific, not for repo) +.claude/ + # Test outputs and reports *.log *.har diff --git "a/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\347\256\241\347\220\206\350\200\205\346\251\237\350\203\275-S3-\346\251\237\351\226\242\343\202\271\343\203\210\343\203\254\343\203\274\343\202\270.ipynb" "b/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\347\256\241\347\220\206\350\200\205\346\251\237\350\203\275-S3-\346\251\237\351\226\242\343\202\271\343\203\210\343\203\254\343\203\274\343\202\270.ipynb" new file mode 100644 index 0000000..682e946 --- /dev/null +++ "b/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\347\256\241\347\220\206\350\200\205\346\251\237\350\203\275-S3-\346\251\237\351\226\242\343\202\271\343\203\210\343\203\254\343\203\274\343\202\270.ipynb" @@ -0,0 +1,1171 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "890e324e-2d56-45e8-976e-677338943ad8", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from getpass import getpass\n", + "\n", + "admin_rdm_url = 'https://admin.test.rdm.example.com/'\n", + "rdm_url = 'https://test.rdm.example.com/'\n", + "idp_name_1 = 'GakuNin RDM IdP'\n", + "\n", + "\n", + "idp_username_1 = None\n", + "idp_password_1 = None\n", + "\n", + "# admin_idp_name を指定すると Embedded DS(IdP選択) 経由でログインする\n", + "admin_idp_name = None\n", + "admin_username = None\n", + "admin_password = None\n", + "# entityID 直接指定（Embedded DS バイパス用。管理者ログインがメール/パスワード形式でない環境でのみ必要）\n", + "admin_idp_entity_id = None\n", + "default_result_path = None\n", + "close_on_fail = False\n", + "transition_timeout = 60000\n", + "skip_failed_test = True\n", + "exclude_notebooks = []\n", + "\n", + "# 機関ストレージ設定用パラメータ\n", + "target_organization = None\n", + "\n", + "# Amazon S3 設定\n", + "s3_access_key = None\n", + "s3_secret_key = None\n", + "s3_bucket = None\n", + "\n", + "# Server Side Encryption: True = Yes\n", + "s3_server_side_encryption = False\n", + "\n", + "# 機関ストレージの表示名（ファイルツリーに表示される名前）\n", + "institutional_storage_name = 'NII Storage'\n", + "\n", + "# プロジェクト名プレフィックス\n", + "rdm_project_prefix = 'TEST-S3-INST-{}'.format(datetime.now().strftime('%Y%m%d-%H%M%S'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2de45df6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# .config.yaml があれば、未設定(None)のパラメータを同名キーで補完する\n", + "# （papermill実行時は先にパラメータが注入されるため、この処理は上書きしない）\n", + "import os as _os\n", + "import yaml as _yaml\n", + "\n", + "_cfg_path = '.config.yaml'\n", + "if _os.path.exists(_cfg_path):\n", + " with open(_cfg_path) as _f:\n", + " _cfg = _yaml.safe_load(_f) or {}\n", + " _loaded = []\n", + " for _k, _v in _cfg.items():\n", + " if _k in globals() and globals()[_k] is None and _v is not None:\n", + " globals()[_k] = _v\n", + " _loaded.append(_k)\n", + " print(f'.config.yaml から補完: {_loaded}')\n", + "else:\n", + " print('.config.yaml なし（プロンプト入力で続行）')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "418932bd-fb02-42f3-a0e5-d7facb342edb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "if idp_username_1 is None:\n", + " idp_username_1 = input(prompt=f'Username for {idp_name_1}')\n", + "if idp_password_1 is None:\n", + " idp_password_1 = getpass(prompt=f'Password for {idp_username_1}@{idp_name_1}')\n", + "if admin_username is None:\n", + " admin_username = input(prompt='Admin Email (管理者画面ログイン用)')\n", + "if admin_password is None:\n", + " admin_password = getpass(prompt=f'Password for {admin_username} (管理者画面)')\n", + "(len(idp_username_1), len(idp_password_1), len(admin_username), len(admin_password))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b8c1c48-52db-4dc6-8eb9-e328c4cb10cd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "if s3_access_key is None:\n", + " s3_access_key = input(prompt='S3 Access Key')\n", + "if s3_secret_key is None:\n", + " s3_secret_key = getpass(prompt='S3 Secret Key')\n", + "if s3_bucket is None:\n", + " s3_bucket = input(prompt='S3 Bucket Name')\n", + "(len(s3_access_key), len(s3_secret_key), len(s3_bucket))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3824eb8a-f08b-4213-8c94-ab3e4f6151c4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import tempfile\n", + "\n", + "work_dir = tempfile.mkdtemp()\n", + "if default_result_path is None:\n", + " default_result_path = work_dir\n", + "work_dir" + ] + }, + { + "cell_type": "markdown", + "id": "bbaf0777-4328-4abc-bdfd-afeb2b513c31", + "metadata": {}, + "source": [ + "# GakuNinRDM 総合テスト [Amazon S3 - 機関ストレージ]\n", + "\n", + "- サブシステム名: 管理者 / ストレージ\n", + "- ページ/アドオン: 機関ストレージ (Amazon S3)\n", + "- 機能分類: 機関ストレージ設定・ファイル操作確認\n", + "- シナリオ名: 管理者で機関ストレージにAmazon S3を設定し、ユーザ画面でプロジェクト作成後ファイルアップロードを確認する\n", + "- 用意するテストデータ: URL一覧、アカウント(既存ユーザー1: GRDM)、Amazon S3のアクセス情報(アクセスキー・シークレットキー・バケット)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53c8bd5f-c6d0-4114-8526-7fe7df529122", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import importlib\n", + "import pandas as pd\n", + "\n", + "import scripts.playwright\n", + "importlib.reload(scripts.playwright)\n", + "\n", + "from scripts.playwright import *\n", + "from scripts import grdm\n", + "\n", + "await init_pw_context(close_on_fail=close_on_fail, last_path=default_result_path)" + ] + }, + { + "cell_type": "markdown", + "id": "cc6be5eb-51bf-4dc9-8660-dcb6a2d1c543", + "metadata": {}, + "source": [ + "## Part 1: 管理者画面での機関ストレージ設定\n", + "\n", + "### GakuNin RDM管理者ページのURLを開く\n", + "\n", + "管理者ページが表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cec2187f-5e72-45a5-8ebd-9c86d249fd55", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import time\n", + "\n", + "async def _step(page):\n", + " await page.goto(admin_rdm_url)\n", + "\n", + " await expect(page.locator('.login-logo')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "4f067da0-2e81-4e24-ae8a-8a17baef89da", + "metadata": {}, + "source": [ + "### ログイン情報を用いてGakuNin RDM管理者画面にログインする\n", + "\n", + "(IdPに関するログイン情報が与えられた場合、)\n", + "GakuNin Embeded DSのプルダウンを展開し、IdPリストから指定されたIdPを選択する。その後、アカウントのID/Passwordを入力して「Login」ボタンを押下する。\n", + "\n", + "(IdPが指定されていない場合、)\n", + "CASのログイン操作を実施する。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9db4190-6ab0-415a-bb79-7e29b0ebc060", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import asyncio\n", + "import urllib.parse\n", + "\n", + "async def _step(page):\n", + " email_field = page.locator('#id_email')\n", + " if await email_field.count() > 0 and await email_field.is_visible():\n", + " # Django メール/パスワードフォーム（enable_form=True の環境）\n", + " await email_field.fill(admin_username)\n", + " await page.locator('#id_password').fill(admin_password)\n", + " await page.locator('//form//button[@type=\"submit\"]').click()\n", + " else:\n", + " # entityID 直接指定で Shibboleth ログインを開始（Embedded DS バイパス）\n", + " if admin_idp_entity_id is None:\n", + " raise ValueError(\n", + " 'admin_idp_entity_id が未設定です。管理者ログインがメール/パスワード形式でない場合は '\n", + " '.config.yaml または本ノートブックのパラメータで対象環境のIdP entityIDを指定してください。'\n", + " )\n", + " base = admin_rdm_url.rstrip('/')\n", + " url = (base + '/Shibboleth.sso/Login'\n", + " '?entityID=' + urllib.parse.quote(admin_idp_entity_id, safe='')\n", + " + '&target=' + urllib.parse.quote(base + '/account/shib-login', safe=''))\n", + " await page.goto(url)\n", + " print('[1] after goto:', page.url)\n", + "\n", + " title = await page.title()\n", + " assert 'Unknown Identity Provider' not in title, (\n", + " f'SPが entityID を認識していません: {admin_idp_entity_id}')\n", + "\n", + " username = page.locator('#username')\n", + " j_username = page.locator('input[name=\"j_username\"]')\n", + " consent = page.locator('#_shib_idp_doNotRememberConsent')\n", + " logout_link = page.locator('//*[@href=\"/account/logout/\"]')\n", + "\n", + " # IdPログインフォーム / 同意画面 / 既ログイン のいずれかを待つ\n", + " await expect(username.or_(j_username).or_(consent).or_(logout_link).first\n", + " ).to_be_visible(timeout=transition_timeout)\n", + "\n", + " if await username.count() > 0 and await username.is_visible():\n", + " await username.fill(admin_username)\n", + " await page.locator('#password').fill(admin_password)\n", + " await page.locator('//button[@type=\"submit\"] | //input[@type=\"submit\"]').first.click()\n", + " print('[2] after credential submit:', page.url)\n", + " elif await j_username.count() > 0 and await j_username.is_visible():\n", + " await j_username.fill(admin_username)\n", + " await page.locator('input[name=\"j_password\"]').fill(admin_password)\n", + " await page.locator('//button[@type=\"submit\"] | //input[@type=\"submit\"]').first.click()\n", + " print('[2] after credential submit (j_username):', page.url)\n", + " else:\n", + " print('[2] ログインフォームなし（既ログイン or 同意画面）:', page.url)\n", + "\n", + " # ログイン失敗の検出（IdPのエラー表示）\n", + " await asyncio.sleep(2)\n", + " err = page.locator('.form-error, .output--error, p.form-element.form-error')\n", + " if await err.count() > 0 and await err.first.is_visible():\n", + " raise AssertionError('IdPログイン失敗: ' + (await err.first.text_content()).strip())\n", + "\n", + " # 同意画面（出ない場合はスキップ）\n", + " try:\n", + " await expect(consent).to_be_visible(timeout=15000)\n", + " await consent.click()\n", + " proceed = page.locator('//*[@name=\"_eventId_proceed\"]')\n", + " await expect(proceed).to_be_enabled()\n", + " await proceed.click()\n", + " print('[3] consent done:', page.url)\n", + " except AssertionError:\n", + " print('[3] 同意画面はスキップ:', page.url)\n", + "\n", + " # ログアウトボタンまたはlogoutリンクが表示されること（日本語/英語環境両対応）\n", + " logout_btn = page.locator('//*[contains(@class, \"btn-danger\") and contains(text(), \"ログアウト\")]')\n", + " logout_link2 = page.locator('//*[@href=\"/account/logout/\"]')\n", + " try:\n", + " await expect(logout_btn.or_(logout_link2)).to_be_visible(timeout=transition_timeout)\n", + " except AssertionError:\n", + " print('[4] 最終確認失敗。現在地:', page.url, '/', await page.title())\n", + " body = await page.evaluate('() => document.body.innerText.substring(0, 800)')\n", + " print(body)\n", + " raise\n", + " print('[4] 管理者画面ログイン成功:', page.url)\n", + "\n", + "await run_pw(_step)\n" + ] + }, + { + "cell_type": "markdown", + "id": "0e1679bc-a65f-4325-832c-3ed637e231a7", + "metadata": {}, + "source": [ + "### サイドメニューの「機関ストレージ」を選択する\n", + "\n", + "機関ストレージの設定画面が表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb73ed05-ef20-4ea0-92cc-6f1624330770", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "async def _goto_inst(page): await page.goto(admin_rdm_url.rstrip('/') + '/custom_storage_location/institutional_storage/')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76e0c4df-11fa-4419-9626-cc25b01994bd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "await run_pw(_goto_inst)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e34ee0d-ebe6-4f9c-a4e8-7ab2bbe5f2b9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import traceback\n", + "\n", + "async def _step(page):\n", + " # サイドメニューの「機関ストレージ」をクリック（「機関ストレージのクォータ」と区別）\n", + " link = page.locator('//a[contains(@href, \"institutional_storage\") and not(contains(@href, \"quota\"))]').first\n", + " await expect(link).to_be_visible(timeout=transition_timeout)\n", + " await link.click()\n", + "\n", + " # 機関ストレージ設定画面が表示されること（ラジオボタンの存在で確認）\n", + " await expect(page.locator('//input[@type=\"radio\"]').first).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "bda312c1-8995-4633-af91-f6b2262a30f7", + "metadata": {}, + "source": [ + "### 機関のリストから対象機関を選択する\n", + "\n", + "対象機関のInstitutional Storage設定画面が表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac7bd974-3285-42ef-b9ff-397d367c9d91", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# async def _step(page):\n", + "# while True:\n", + "# link = page.locator(f'//a[text() = \"{target_organization}\"]')\n", + "# try:\n", + "# await expect(link).to_be_visible()\n", + "# except:\n", + "# traceback.print_exc()\n", + "# print('Search next page...')\n", + "# next_button = page.locator('//a[i[contains(@class, \"fa-angle-right\")]]')\n", + "# if await next_button.count() == 0 or await next_button.is_disabled():\n", + "# raise Exception(f'Organization \"{target_organization}\" not found in any page')\n", + "# await next_button.click()\n", + "# await expect(page.locator('//h2[contains(text(), \"List of Institutions\") or contains(text(), \"機関のリスト\")]')).to_be_visible(timeout=transition_timeout)\n", + "# continue\n", + "# await link.click()\n", + "# break\n", + "\n", + "# await expect(page.locator('//h2[contains(text(), \"Institutional Storage\")]')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "# await run_pw(_step)\n" + ] + }, + { + "cell_type": "markdown", + "id": "5a2732ce-d7dd-425a-88f5-4e65424ffef8", + "metadata": {}, + "source": [ + "### Amazon S3 を選択する\n", + "\n", + "Amazon S3 のラジオボタンが選択されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba1b98f0-106a-418f-bfb6-c6b96a696f4e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "async def _step(page):\n", + " # Amazon S3 のラジオボタンを選択\n", + " # （既定で別プロバイダ(例: s3compatsigv4)が選択済みの場合があるため、check + 選択状態を検証する）\n", + " radio = page.locator('//input[@type=\"radio\" and @value=\"s3\"]')\n", + " await expect(radio).to_be_visible(timeout=transition_timeout)\n", + " await radio.scroll_into_view_if_needed()\n", + " try:\n", + " await radio.check()\n", + " except Exception:\n", + " await radio.check(force=True)\n", + " await expect(radio).to_be_checked()\n", + " selected = await page.evaluate('() => { const r = document.querySelector(\"input[name=options]:checked\"); return r ? r.value : \"none\"; }')\n", + " print('selected provider:', selected)\n", + " assert selected == 's3', f'Amazon S3 を選択できていません: {selected}'\n", + "\n", + "await run_pw(_step)\n" + ] + }, + { + "cell_type": "markdown", + "id": "3be35474-bb07-45e8-8dba-299587c246e8", + "metadata": {}, + "source": [ + "### 「Save」ボタンをクリックし、確認ダイアログで確認文字列を入力して「Change」をクリックする\n", + "\n", + "機関ストレージ変更の確認ダイアログが表示される。表示された確認文字列を入力し、「Change」をクリックするとAmazon S3の設定モーダルが表示されること。\n", + "\n", + "※ 確認文字列はランダムに生成されるため、画面を確認して手入力する。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9d41a17-c5f4-4d09-b692-0fa23405314e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import asyncio\n", + "\n", + "async def _step(page):\n", + " global s3_modal_opened\n", + " s3_modal_opened = False\n", + "\n", + " # storage_name フィールドの状態を確認し、必要なら入力する\n", + " storage_name = page.locator('#storage_name')\n", + " sn_visible = await storage_name.is_visible()\n", + " sn_enabled = await storage_name.is_enabled() if sn_visible else False\n", + " sn_value = await storage_name.input_value() if sn_visible else ''\n", + " print(f'storage_name: visible={sn_visible}, enabled={sn_enabled}, value=\"{sn_value}\"')\n", + " if sn_visible and sn_enabled and not sn_value.strip():\n", + " await storage_name.fill(institutional_storage_name)\n", + " print(f'Filled storage_name: {institutional_storage_name}')\n", + "\n", + " # Save(保存) ボタンをクリック\n", + " save_btn = page.locator('//button[@type=\"submit\" and contains(@class, \"btn-success\")]').first\n", + " await save_btn.scroll_into_view_if_needed()\n", + " await save_btn.click()\n", + " await asyncio.sleep(2)\n", + "\n", + " confirm_text = page.locator('#bbConfirmText')\n", + " s3_modal = page.locator('#s3_modal')\n", + "\n", + " # 確認ダイアログ(bootbox)が出た場合は確認文字列を入力して変更を確定する\n", + " try:\n", + " await expect(confirm_text).to_be_visible(timeout=10000)\n", + " confirm_strong = page.locator('//div[contains(@class, \"bootbox-body\")]//strong')\n", + " confirmation_string = await confirm_strong.text_content()\n", + " print(f'Confirmation string: {confirmation_string}')\n", + " await confirm_text.fill(confirmation_string)\n", + " await page.locator('//div[contains(@class, \"modal\") and contains(@class, \"bootbox\")]//button[contains(@class, \"btn-danger\")]').click()\n", + " print('確認ダイアログを処理しました')\n", + " except:\n", + " print('確認ダイアログは表示されませんでした（変更なしの可能性）')\n", + "\n", + " # Amazon S3 認証モーダルが開いたか判定する（開かなければ既設定とみなしスキップ）\n", + " try:\n", + " await expect(s3_modal).to_be_visible(timeout=20000)\n", + " s3_modal_opened = True\n", + " print('Amazon S3 認証モーダルが表示されました（認証情報を入力します）')\n", + " except:\n", + " s3_modal_opened = False\n", + " selected = await page.evaluate('() => { var r = document.querySelector(\"input[name=options]:checked\"); return r ? r.value : \"none\"; }')\n", + " print(f'Amazon S3 認証モーダルは表示されませんでした。既に機関ストレージ設定済み(selectedProvider={selected})とみなし、認証情報入力〜保存のセルをスキップします。')\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "2595e57f-3c1f-4ef7-857f-8056df221193", + "metadata": {}, + "source": [ + "### Access Key、Secret Key、Bucketを入力し、Server Side EncryptionをYesに設定する\n", + "\n", + "各項目が正しく入力されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97aca39c-8969-40f8-aa83-400909b5f299", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import asyncio\n", + "\n", + "async def _step(page):\n", + " if not s3_modal_opened:\n", + " print('S3モーダル未表示のためスキップ（既設定）')\n", + " return\n", + " # Access Key を入力\n", + " await page.locator('#s3_access_key').fill(s3_access_key)\n", + "\n", + " # Secret Key を入力\n", + " await page.locator('#s3_secret_key').fill(s3_secret_key)\n", + "\n", + " # Bucket を入力\n", + " await page.locator('#s3_bucket').fill(s3_bucket)\n", + "\n", + " # Server Side Encryption のチェックボックス\n", + " # sse_checkbox = page.locator('#s3_server_side_encryption')\n", + " # is_checked = await sse_checkbox.is_checked()\n", + " # if s3_server_side_encryption and not is_checked:\n", + " # await sse_checkbox.click()\n", + " # elif not s3_server_side_encryption and is_checked:\n", + " # await sse_checkbox.click()\n", + "\n", + " # keyupイベントを発火させてバリデーションをトリガー\n", + " await page.evaluate('() => { document.querySelectorAll(\"#s3_modal input\").forEach(el => el.dispatchEvent(new Event(\"keyup\", { bubbles: true }))); }')\n", + "\n", + " # 入力後少し待つ\n", + " await asyncio.sleep(1)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "c3dee5f8-1791-467a-bb98-b3849f65070c", + "metadata": {}, + "source": [ + "### 「Connect」ボタンをクリックして接続テストを行う\n", + "\n", + "接続テストが成功すること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "378f2883-4828-4a08-821f-5cc6aa7d61e6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "async def _step(page):\n", + " if not s3_modal_opened:\n", + " print('S3モーダル未表示のためスキップ（既設定）')\n", + " return\n", + " # Connect ボタンをクリック\n", + " connect_btn = page.locator('#s3_connect')\n", + " await expect(connect_btn).to_be_enabled(timeout=transition_timeout)\n", + " await connect_btn.click()\n", + "\n", + " # Save ボタンが有効になるのを待つ（接続成功の証）\n", + " save_btn = page.locator('#s3_save')\n", + " await expect(save_btn).to_be_enabled(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "8860c955-50c1-4c43-a3b6-d167cf4327db", + "metadata": {}, + "source": [ + "### 「Save」ボタンをクリックして設定を保存する\n", + "\n", + "機関ストレージの設定が保存されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ad1d8bd-0296-4937-954a-dbcadd1096e2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import scripts.playwright as _sp\n", + "if s3_modal_opened:\n", + " _msg = await _sp.current_contexts[-1][1][-1].evaluate('() => document.getElementById(\"s3_message\") ? document.getElementById(\"s3_message\").innerText : \"no message element\"'); print('server message:', _msg)\n", + "else:\n", + " print('S3モーダル未表示のため server message 確認をスキップ（既設定）')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4561ee1-2f62-45ef-bd75-cdda843cb1d6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import scripts.playwright as _sp\n", + "if s3_modal_opened:\n", + " _r = await _sp.current_contexts[-1][1][-1].evaluate('async () => { const inst = (window.contextVars && window.contextVars.institution_id) || \"\"; const params = {provider_short_name: \"s3\", s3_access_key: document.getElementById(\"s3_access_key\").value, s3_secret_key: document.getElementById(\"s3_secret_key\").value, s3_bucket: document.getElementById(\"s3_bucket\").value, s3_server_side_encryption: document.getElementById(\"s3_server_side_encryption\").value}; const r = await fetch(\"/custom_storage_location/test_connection/\" + inst, {method: \"POST\", headers: {\"Content-Type\": \"application/json; charset=utf-8\", \"X-CSRFToken\": document.querySelector(\"[name=csrfmiddlewaretoken]\").value}, body: JSON.stringify(params), credentials: \"same-origin\"}); const t = await r.text(); return \"status=\" + r.status + \" body=\" + t.substring(0, 600); }'); print(_r)\n", + "else:\n", + " print('S3モーダル未表示のため test_connection 検証をスキップ（既設定）')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a93959e-4273-4183-bbce-28589b7d09bc", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "async def _step(page):\n", + " if not s3_modal_opened:\n", + " print('S3モーダル未表示のためスキップ（既設定）')\n", + " return\n", + " # Save ボタンをクリック\n", + " save_btn = page.locator('#s3_save')\n", + " await save_btn.click()\n", + "\n", + " # モーダルが閉じるのを待つ\n", + " await expect(page.locator('#s3_modal')).to_be_hidden(timeout=transition_timeout)\n", + "\n", + " # 少し待って設定が反映されるのを確認\n", + " await asyncio.sleep(2)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "6a954f1b-5931-4546-88a8-3823fe58c2eb", + "metadata": {}, + "source": [ + "### 管理者画面からログアウトする\n", + "\n", + "ログアウトが完了すること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6ca069b-ad5f-421b-a292-1e0685c343ad", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "async def _step(page):\n", + " # ログアウト（日本語/英語環境両対応）\n", + " logout_btn = page.locator('//*[contains(@class, \"btn-danger\") and contains(text(), \"ログアウト\")]')\n", + " logout_link = page.locator('//*[@href=\"/account/logout/\"]')\n", + " if await logout_btn.count() > 0:\n", + " await logout_btn.click()\n", + " else:\n", + " await logout_link.click()\n", + "\n", + " await expect(page.locator('.login-logo')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "abb6c1d2-3d72-4e04-8e35-ae9eb8de3def", + "metadata": {}, + "source": [ + "## Part 2: ユーザ画面でのプロジェクト作成とファイルアップロード\n", + "\n", + "### GRDMトップページを表示する\n", + "\n", + "GRDMトップページが表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f598e7b-c149-4fdb-9ba6-6ecb20ef01f8", + "metadata": {}, + "outputs": [], + "source": [ + "import scripts.grdm\n", + "import importlib\n", + "importlib.reload(scripts.grdm)\n", + "\n", + "\n", + "async def _step(page):\n", + " await page.goto(rdm_url)\n", + "\n", + " # ページの読み込みを待つ\n", + " await page.wait_for_load_state('networkidle')\n", + "\n", + " # 同意するボタンがあれば先にクリック\n", + " consent_button = page.locator('//button[text() = \"同意する\"]')\n", + " if await consent_button.count() > 0 and await consent_button.is_visible():\n", + " await consent_button.click()\n", + " await page.wait_for_load_state('networkidle')\n", + "\n", + " # 未ログインのGRDMトップページ（WAYF/Embedded DS）が表示されていることを確認する\n", + " # 実際のログインは次セルの scripts.grdm.login で行う\n", + " await scripts.grdm.expect_anonymous_toppage(page, idp_name_1, transition_timeout=transition_timeout)\n", + "\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "b6e5deb9-57ff-4b39-889d-afc57504ed23", + "metadata": {}, + "source": [ + "### ログイン情報を用いてGakuNin RDMにログインする\n", + "\n", + "(IdPに関するログイン情報が与えられた場合、)\n", + "GakuNin Embeded DSのプルダウンを展開し、IdPリストから指定されたIdPを選択する。その後、アカウントのID/Passwordを入力して「Login」ボタンを押下する。\n", + "\n", + "(IdPが指定されていない場合、)\n", + "CASのログイン操作を実施する。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f591bebe-3a0f-4b63-adb4-741f15c8c82b", + "metadata": {}, + "outputs": [], + "source": [ + "import scripts.grdm\n", + "importlib.reload(scripts.grdm)\n", + "\n", + "async def _step(page):\n", + " await scripts.grdm.login(\n", + " page, idp_name_1, idp_username_1, idp_password_1, transition_timeout=transition_timeout\n", + " )\n", + "\n", + " # ダッシュボードが表示されること（プロジェクト作成ボタンで確認）\n", + " await expect(page.locator('//*[@data-test-create-project-modal-button]')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "a8288a84-8fa6-4e1c-95ac-b59cca649f46", + "metadata": {}, + "source": [ + "### プロジェクトを作成する\n", + "\n", + "「{rdm_project_prefix}-institutional」プロジェクトが作成されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2439f85-8a2b-4997-b514-307283ffe893", + "metadata": {}, + "outputs": [], + "source": [ + "import scripts.grdm\n", + "importlib.reload(scripts.grdm)\n", + "\n", + "rdm_project_name = f'{rdm_project_prefix}-institutional'\n", + "\n", + "async def _step(page):\n", + " await expect(page.locator('//*[@data-test-create-project-modal-button]')).to_have_count(1)\n", + "\n", + " await scripts.grdm.ensure_project_exists(page, rdm_project_name, transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "d0cd996a-f08a-476a-92f1-cde36abc27ae", + "metadata": {}, + "source": [ + "### ダッシュボードのプロジェクト一覧から作成したプロジェクトをクリックする\n", + "\n", + "プロジェクトダッシュボードが表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18aacbc0-4187-4706-b0c5-8a00d9e35d35", + "metadata": {}, + "outputs": [], + "source": [ + "async def _step(page):\n", + " await page.locator(f'//*[@data-test-dashboard-item-title and text()=\"{rdm_project_name}\"]').click()\n", + "\n", + " # プロジェクトダッシュボードのナビタブが表示されること\n", + " await expect(page.locator('#projectNavFiles')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "3e72dbec-6f1e-4d93-9bf3-bc7387375ccc", + "metadata": {}, + "source": [ + "### 「ファイル」タブをクリックする\n", + "\n", + "ファイルタブが表示されること。NII Storageの代わりにAmazon S3の機関ストレージが表示されること。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6198b46f-c596-4bab-a491-10fec7b6be15", + "metadata": {}, + "outputs": [], + "source": [ + "async def _step(page):\n", + " await page.locator('#projectNavFiles a').click()\n", + "\n", + " await expect(page.locator('//*[@id = \"treeGrid\"]')).to_be_visible(timeout=transition_timeout)\n", + "\n", + " # 少し待ってファイルツリーが読み込まれるのを確認\n", + " await asyncio.sleep(3)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "eebda761-0dba-4fa1-a58f-7301c01ec39e", + "metadata": {}, + "source": [ + "### テスト用ファイルを作成する\n", + "\n", + "テスト用のファイルが作成されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e46b1c0-30b3-4f01-8131-f016b481afe8", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "test_file_name = 'test_institutional_storage.txt'\n", + "test_file_path = os.path.join(work_dir, test_file_name)\n", + "with open(test_file_path, 'w') as f:\n", + " f.write(f'Institutional Storage Test - Amazon S3\\n')\n", + " f.write(f'Created at: {datetime.now().isoformat()}\\n')\n", + " f.write(f'Bucket: {s3_bucket}\\n')\n", + "\n", + "print(f'Test file created: {test_file_path}')" + ] + }, + { + "cell_type": "markdown", + "id": "8d30a467-dbfa-41ad-8ceb-139dbefff42f", + "metadata": {}, + "source": [ + "### ファイルをアップロードする\n", + "\n", + "ファイルがアップロードされ、ファイル一覧に表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa4930e0-7afb-43ec-b92d-9e08423c5102", + "metadata": {}, + "outputs": [], + "source": [ + "async def _step(page):\n", + " # 機関ストレージプロバイダをクリックして開く\n", + " await page.locator(f'//*[contains(text(), \"{institutional_storage_name}\")]').click()\n", + " await asyncio.sleep(2)\n", + "\n", + " # アップロードボタンが表示されるのを待つ\n", + " upload_btn = page.locator('//i[contains(@class, \"fa-upload\")]/../*[text() = \"アップロード\"]')\n", + " await expect(upload_btn).to_be_visible(timeout=transition_timeout)\n", + "\n", + " await scripts.grdm.upload_file(page, test_file_path)\n", + "\n", + " # アップロードしたファイルが表示されるのを確認\n", + " await expect(page.locator(f'//*[contains(text(), \"{test_file_name}\")]')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "9069b114-dec1-4366-8734-c1935205fd6b", + "metadata": {}, + "source": [ + "### アップロードしたファイルをクリックして内容を確認する\n", + "\n", + "ファイルの内容が表示されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "998fd6eb-f4cd-482a-9e39-9b834a91f988", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "async def _step(page):\n", + " if 'files' in page.url and test_file_name in await page.title():\n", + " pass\n", + " else:\n", + " await grdm.get_select_file_title_locator(page, test_file_name).click(timeout=transition_timeout)\n", + " await asyncio.sleep(3)\n", + "\n", + " # ファイル詳細ページが表示されていることを確認（ファイル名ヘッダーの拡張子）\n", + " filename_body, filename_ext = os.path.splitext(test_file_name)\n", + " await expect(page.locator(f'//h2[contains(text(), \"{filename_body}\")]//*[@id = \"file-ext\"]')).to_have_text(filename_ext, timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "8e4617ad-eaf5-4c6b-8522-80b566de8a3c", + "metadata": {}, + "source": [ + "### プロジェクトを削除する\n", + "\n", + "プロジェクトが削除されること" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cf5498e-f791-4906-9e55-f278ff9a6be0", + "metadata": {}, + "outputs": [], + "source": [ + "async def _step(page):\n", + " await scripts.grdm.delete_project(page)\n", + "\n", + " # ダッシュボードに戻ったことを確認\n", + " await expect(page.locator('//*[@data-test-create-project-modal-button]')).to_be_visible(timeout=transition_timeout)\n", + "\n", + "await run_pw(_step)" + ] + }, + { + "cell_type": "markdown", + "id": "b4cb3797-1a59-46f0-bdd1-b2e33b2fb54f", + "metadata": {}, + "source": [ + "## Part 3: 後処理\n", + "\n", + "### 「ファイル基本操作」テストの実施（機関ストレージ利用時）\n", + "\n", + "テスト「テスト手順-ストレージ共通-ファイル基本操作」を機関ストレージ設定下でプロジェクトダッシュボードにて実施する。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c70ad802-c472-40db-80e6-1482425f20a6", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "import os\n", + "import papermill as pm\n", + "import traceback\n", + "from scripts.papermillHelpers import gen_run_notebook\n", + "\n", + "def make_result_dir(base_path):\n", + " result_dir = os.path.join(base_path, 'notebooks')\n", + " os.makedirs(result_dir, exist_ok=True)\n", + " return result_dir\n", + "\n", + "result_dir = make_result_dir(default_result_path)\n", + "\n", + "run_notebook = gen_run_notebook(\n", + " result_dir,\n", + " transition_timeout,\n", + " dict(\n", + " rdm_url=rdm_url,\n", + " idp_name_1=idp_name_1,\n", + " idp_username_1=idp_username_1,\n", + " idp_password_1=idp_password_1,\n", + " ),\n", + " skip_failed_test,\n", + " exclude_notebooks,\n", + ")\n", + "\n", + "result_notebooks = []\n", + "result_dir" + ] + }, + { + "cell_type": "markdown", + "id": "a2e6f5f5-a47b-4c93-871c-9911f18c5fc8", + "metadata": {}, + "source": [ + "### プロジェクトダッシュボードでの「ファイル基本操作」テストの実施\n", + "\n", + "テスト「テスト手順-ストレージ共通-ファイル基本操作」をプロジェクトダッシュボードで実施する。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c147c50c-7dc4-4be7-8fbb-d98197761f23", + "metadata": {}, + "outputs": [], + "source": [ + "result_notebooks.append(run_notebook(\n", + " 'テスト手順-ストレージ共通-ファイル基本操作.ipynb',\n", + " dict(\n", + " enable_52gb_file_upload=False,\n", + " target_storage_name=institutional_storage_name,\n", + " target_storage_id='osfstorage',\n", + " target_file_view='project-dashboard',\n", + " rdm_project_name=f'{rdm_project_prefix}-dashboard',\n", + " ),\n", + "))\n", + "result_notebooks" + ] + }, + { + "cell_type": "markdown", + "id": "92c54c10-cfd8-4015-a137-ca7e16c7694f", + "metadata": {}, + "source": [ + "### ファイルタブでの「ファイル基本操作」テストの実施\n", + "\n", + "テスト「テスト手順-ストレージ共通-ファイル基本操作」をファイルタブで実施する。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce8d4388-67ab-45de-9a56-fa875222b292", + "metadata": {}, + "outputs": [], + "source": [ + "result_notebooks.append(run_notebook(\n", + " 'テスト手順-ストレージ共通-ファイル基本操作.ipynb',\n", + " dict(\n", + " enable_52gb_file_upload=False,\n", + " target_storage_name=institutional_storage_name,\n", + " target_storage_id='osfstorage',\n", + " target_file_view='file-tab',\n", + " rdm_project_name=f'{rdm_project_prefix}-filetab',\n", + " ),\n", + " '-file-tab',\n", + "))\n", + "result_notebooks" + ] + }, + { + "cell_type": "markdown", + "id": "e66bd529-4264-4d41-ab6d-9c8e890f40a7", + "metadata": {}, + "source": [ + "終了処理を実施。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55a1972b-8445-40af-9080-f7e7b4ddb40b", + "metadata": {}, + "outputs": [], + "source": [ + "await finish_pw_context(timeout=300)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45a23f07-21ea-4438-b045-a9df3a36de1b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!rm -fr {work_dir}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2896b057-e9f0-44cf-a494-ea6dac380477", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import scripts.playwright as _sp; print('contexts=', None if _sp.current_contexts is None else len(_sp.current_contexts)); await _sp.finish_pw_context(timeout=300)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "693d7734-6fe4-4428-875f-e568b2209027", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git "a/\345\217\226\343\202\212\343\201\276\343\201\250\343\202\201-S3\345\205\261\351\200\232.ipynb" "b/\345\217\226\343\202\212\343\201\276\343\201\250\343\202\201-S3\345\205\261\351\200\232.ipynb" index 19ae26e..14e1325 100644 --- "a/\345\217\226\343\202\212\343\201\276\343\201\250\343\202\201-S3\345\205\261\351\200\232.ipynb" +++ "b/\345\217\226\343\202\212\343\201\276\343\201\250\343\202\201-S3\345\205\261\351\200\232.ipynb" @@ -71,6 +71,31 @@ "rdm_project_prefix = 'TEST-{}-{}'.format(target_storage_id.upper(), datetime.now().strftime('%Y%m%d-%H%M%S'))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# .config.yaml があれば、未設定(None)のパラメータを同名キーで補完する\n", + "# （papermill実行時は先にパラメータが注入されるため、この処理は上書きしない）\n", + "import os as _os\n", + "import yaml as _yaml\n", + "\n", + "_cfg_path = '.config.yaml'\n", + "if _os.path.exists(_cfg_path):\n", + " with open(_cfg_path) as _f:\n", + " _cfg = _yaml.safe_load(_f) or {}\n", + " _loaded = []\n", + " for _k, _v in _cfg.items():\n", + " if _k in globals() and globals()[_k] is None and _v is not None:\n", + " globals()[_k] = _v\n", + " _loaded.append(_k)\n", + " print(f'.config.yaml から補完: {_loaded}')\n", + "else:\n", + " print('.config.yaml なし（プロンプト入力で続行）')\n" + ] + }, { "cell_type": "code", "execution_count": null, From 25c5fa7df0136b69fc4a7ba388b647d266d40f93 Mon Sep 17 00:00:00 2001 From: tishin-endou Date: Tue, 21 Jul 2026 10:01:46 +0900 Subject: [PATCH 4/4] =?UTF-8?q?ci(s3):=20AWS=20S3=E3=83=AA=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=83=A7=E3=83=B3=E3=83=86=E3=82=B9=E3=83=88=E3=82=92?= =?UTF-8?q?=E6=97=A2=E5=AE=9A=E3=81=A7=E7=84=A1=E5=8A=B9=E5=8C=96=EF=BC=88?= =?UTF-8?q?=E3=83=91=E3=83=96=E3=83=AA=E3=83=83=E3=82=AF=E3=83=AA=E3=83=9D?= =?UTF-8?q?=E3=82=B8=E3=83=88=E3=83=AA=E5=AF=BE=E5=BF=9C=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit user-aws-s3 マトリクス項目は実AWS認証情報のGitHub Secretsを必要とするが、 パブリックリポジトリではSecretを登録できず、有効なままだと認証情報未設定で CIが失敗する。そのため既定でコメントアウトし、ローカル/Secret設定済みforkで テストする場合のみ有効化する旨をコメントで明記した。 generate_ci_config.sh 側の --aws-s3 処理は温存（有効化時にそのまま使える）。 --- .github/workflows/e2e-test.yml | 36 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 7f7031c..3b09801 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -162,19 +162,29 @@ jobs: weko_enabled: false jupyterhub_enabled: false flowable_enabled: false - - name: user-aws-s3 - display_name: "User Tests (AWS S3 SigV4 Regions)" - include_admin: false - skip_admin: true - skip_metadata: true - skip_default_storage: true - skip_login: true - skip_130mb_upload: true - minio_enabled: false - aws_s3_enabled: true - weko_enabled: false - jupyterhub_enabled: false - flowable_enabled: false + # AWS S3 (SigV4リージョン) の実接続テスト。 + # このジョブは実AWSのアクセスキー等を GitHub Secrets + # (AWS_S3_ACCESS_KEY_1/2, AWS_S3_SECRET_KEY_1/2, + # AWS_S3_LEGACY_REGION, AWS_S3_LEGACY_BUCKET_NAME, + # AWS_S3_V4_REGION, AWS_S3_V4_BUCKET_NAME) + # から取得する。パブリックリポジトリでは Secret を登録できないため + # 既定では無効化している（有効なままだと認証情報未設定でCIが失敗する）。 + # + # ローカル / Secret を設定済みのプライベートfork でテストする場合のみ、 + # 以下のブロックのコメントを解除して有効化すること。 + # - name: user-aws-s3 + # display_name: "User Tests (AWS S3 SigV4 Regions)" + # include_admin: false + # skip_admin: true + # skip_metadata: true + # skip_default_storage: true + # skip_login: true + # skip_130mb_upload: true + # minio_enabled: false + # aws_s3_enabled: true + # weko_enabled: false + # jupyterhub_enabled: false + # flowable_enabled: false - name: admin-minio display_name: "Admin Tests (MinIO)" include_admin: true