Merge upstream/main into feature/new-metadata-ui

Author: yacchin1205

.github/scripts/generate_ci_config.sh

@@ -3,7 +3,7 @@ set -xeuo pipefail
 
 if [[ $# -lt 2 ]]; then
   cat >&2 <<'USAGE'
-Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub] [--weko] [--flowable]
+Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst]
 USAGE
   exit 1
 fi
@@ -15,6 +15,8 @@ MINIO=false
 JUPYTERHUB=false
 WEKO=false
 FLOWABLE=false
+S3COMPATSIGV4=false
+S3COMPATSIGV4_INST=false
 
 for arg in "$@"; do
   case "$arg" in
@@ -30,6 +32,12 @@ for arg in "$@"; do
     --flowable)
       FLOWABLE=true
       ;;
+    --s3compatsigv4)
+      S3COMPATSIGV4=true
+      ;;
+    --s3compatsigv4-inst)
+      S3COMPATSIGV4_INST=true
+      ;;
     *)
       echo "Unknown argument: ${arg}" >&2
       exit 1
@@ -146,4 +154,45 @@ workflow_enabled: false
 EOF
 fi
 
+if [[ "${S3COMPATSIGV4}" == "true" ]]; then
+  if [[ -z "${S3COMPATSIGV4_ACCESS_KEY_1:-}" || -z "${S3COMPATSIGV4_SECRET_KEY_1:-}" ]]; then
+    echo "S3 compat sigv4 credentials are not set" >&2
+    exit 1
+  fi
+
+  cat >> "${OUTPUT}" <<EOF
+
+s3compatsigv4_enabled: true
+
+s3compatsigv4_access_key_1: '${S3COMPATSIGV4_ACCESS_KEY_1}'
+s3compatsigv4_secret_access_key_1: '${S3COMPATSIGV4_SECRET_KEY_1}'
+s3compatsigv4_endpoint_1: '${S3COMPATSIGV4_ENDPOINT}'
+s3compatsigv4_test_bucket_name_1: '${S3COMPATSIGV4_BUCKET_NAME_1}'
+
+s3compatsigv4_access_key_2: '${S3COMPATSIGV4_ACCESS_KEY_2}'
+s3compatsigv4_secret_access_key_2: '${S3COMPATSIGV4_SECRET_KEY_2}'
+s3compatsigv4_endpoint_2: '${S3COMPATSIGV4_ENDPOINT}'
+s3compatsigv4_test_bucket_name_2: '${S3COMPATSIGV4_BUCKET_NAME_2}'
+
+s3compatsigv4_type_name_1: '${S3COMPATSIGV4_SERVICE_NAME}'
+s3compatsigv4_type_name_2: '${S3COMPATSIGV4_SERVICE_NAME}'
+EOF
+fi
+
+if [[ "${S3COMPATSIGV4_INST}" == "true" ]]; then
+  if [[ -z "${S3COMPATSIGV4_INST_ACCESS_KEY:-}" || -z "${S3COMPATSIGV4_INST_SECRET_KEY:-}" ]]; then
+    echo "S3 compat sigv4 institutional storage credentials are not set" >&2
+    exit 1
+  fi
+
+  cat >> "${OUTPUT}" <<EOF
+
+s3compatsigv4_institutional_storage_enabled: true
+s3compatsigv4_inst_endpoint_url: '${S3COMPATSIGV4_INST_ENDPOINT}'
+s3compatsigv4_inst_access_key: '${S3COMPATSIGV4_INST_ACCESS_KEY}'
+s3compatsigv4_inst_secret_key: '${S3COMPATSIGV4_INST_SECRET_KEY}'
+s3compatsigv4_inst_bucket: '${S3COMPATSIGV4_INST_BUCKET}'
+EOF
+fi
+
 cat "${OUTPUT}"

.github/scripts/setup_minio.sh

@@ -109,4 +109,50 @@ PY
 
 mv "${tmp_json}" "${settings_json}"
 
+# Also register MinIO service for s3compatsigv4 addon if available
+sigv4_settings_json="${RDM_ROOT}/addons/s3compatsigv4/static/settings.json"
+
+if [[ -f "${sigv4_settings_json}" ]]; then
+  tmp_json_sigv4=$(mktemp)
+
+  python - "$sigv4_settings_json" "$tmp_json_sigv4" <<'PY'
+import json
+import sys
+
+src, dst = sys.argv[1:3]
+with open(src) as f:
+    data = json.load(f)
+
+service_name = "MinIO (CI)"
+host = "minio:9000"
+
+available_services = data.setdefault("availableServices", [])
+
+if not any(s.get("name") == service_name for s in available_services):
+    available_services.append({
+        "name": service_name,
+        "host": host,
+        "bucketLocations": {
+            "us-east-1": {
+                "name": "us-east-1",
+                "host": host,
+            },
+            "": {
+                "name": "us-east-1",
+            },
+        },
+        "serverSideEncryption": False,
+    })
+
+with open(dst, "w") as f:
+    json.dump(data, f, ensure_ascii=False, indent=4)
+    f.write("\n")
+PY
+
+  mv "${tmp_json_sigv4}" "${sigv4_settings_json}"
+  echo "MinIO configuration applied for s3compatsigv4"
+else
+  echo "WARNING: s3compatsigv4 settings.json not found: ${sigv4_settings_json} (skipping)"
+fi
+
 echo "MinIO configuration applied"

.github/workflows/e2e-test.yml

@@ -43,6 +43,14 @@ on:
         description: 'Custom WaterButler image (default: niicloudoperation/rdm-waterbutler:latest)'
         required: false
         default: ''
+      wb_repository:
+        description: 'RDM-waterbutler repository (default: RCOSDP/RDM-waterbutler)'
+        required: false
+        default: ''
+      wb_branch:
+        description: 'RDM-waterbutler branch to build and test against'
+        required: false
+        default: ''
       tljh_version:
         description: 'TLJH version (default: 1.0.0)'
         required: false
@@ -165,6 +173,20 @@ jobs:
             skip_login: true
             skip_130mb_upload: true
             minio_enabled: true
+            s3compatsigv4_enabled: true
+            weko_enabled: false
+            jupyterhub_enabled: false
+            flowable_enabled: false
+          - name: admin-minio
+            display_name: "Admin Tests (MinIO)"
+            include_admin: true
+            skip_admin: true
+            skip_metadata: true
+            skip_default_storage: true
+            skip_login: true
+            skip_130mb_upload: true
+            minio_enabled: true
+            s3compatsigv4_institutional_storage: true
             weko_enabled: false
             jupyterhub_enabled: false
             flowable_enabled: false
@@ -249,6 +271,8 @@ jobs:
     name: E2E ${{ matrix.test-group.display_name }}
     env:
       MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }}
+      S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }}
+      S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }}
       WEKO_ENABLED: ${{ matrix.test-group.weko_enabled == true && 'true' || 'false' }}
       JUPYTERHUB_ENABLED: ${{ matrix.test-group.jupyterhub_enabled == true && 'true' || 'false' }}
       FLOWABLE_ENABLED: ${{ matrix.test-group.flowable_enabled == true && 'true' || 'false' }}
@@ -639,6 +663,46 @@ jobs:
         export INCLUDE_ADMIN=${{ matrix.test-group.include_admin }}
         setup_config_files
 
+    - name: Parse PR body and workflow parameters
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          cat > /tmp/pr_body.txt << 'EOF'
+        ${{ github.event.pull_request.body }}
+        EOF
+          echo "WB_CUSTOM_REPO=$(grep -oP '^-\s*WB_REPOSITORY:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true)" >> $GITHUB_ENV
+          echo "WB_CUSTOM_BRANCH=$(grep -oP '^-\s*WB_BRANCH:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true)" >> $GITHUB_ENV
+        else
+          echo "WB_CUSTOM_REPO=${{ github.event.inputs.wb_repository }}" >> $GITHUB_ENV
+          echo "WB_CUSTOM_BRANCH=${{ github.event.inputs.wb_branch }}" >> $GITHUB_ENV
+        fi
+
+    - name: Build custom WaterButler image
+      if: env.WB_CUSTOM_BRANCH != ''
+      run: |
+        WB_REPO="${WB_CUSTOM_REPO:-RCOSDP/RDM-waterbutler}"
+        WB_BRANCH="${WB_CUSTOM_BRANCH}"
+        WB_BASE_BRANCH="develop"
+
+        echo "Building custom WaterButler image from ${WB_REPO}@${WB_BRANCH}"
+
+        # Clone base branch
+        git clone --depth=50 -b "$WB_BASE_BRANCH" "https://github.com/${WB_REPO}.git" RDM-waterbutler
+        cd RDM-waterbutler
+
+        # Merge custom branch
+        git fetch origin "$WB_BRANCH"
+        git config user.name "github-actions[bot]"
+        git config user.email "RDM-e2e-test-nb@example.com"
+        git merge --no-ff FETCH_HEAD -m "Merge ${WB_BRANCH} for E2E tests"
+
+        # Build Docker image
+        docker build -t rdm-waterbutler:custom .
+        cd ..
+
+        # Override WB_IMAGE for subsequent steps
+        echo "WB_IMAGE=rdm-waterbutler:custom" >> $GITHUB_ENV
+        echo "Custom WaterButler image built successfully"
+
     - name: Create docker-compose override for NII Cloud Operation images
       working-directory: RDM-osf.io
       run: |
@@ -668,22 +732,31 @@ jobs:
           echo "TLJH_PLUGIN_OVERRIDE=$TLJH_PLUGIN_OVERRIDE" >> $GITHUB_ENV
           echo "TLJH_REPO2DOCKER_IMAGE_OVERRIDE=$TLJH_REPO2DOCKER_IMAGE_OVERRIDE" >> $GITHUB_ENV
           echo "TLJH_RDMFS_IMAGE_OVERRIDE=$TLJH_RDMFS_IMAGE_OVERRIDE" >> $GITHUB_ENV
-          
+
           # Use overrides if found, otherwise defaults
+          # If WB_IMAGE was set by "Build custom WaterButler image" step, use it
           export OSF_IMAGE="${OSF_OVERRIDE:-niicloudoperation/rdm-osf.io:latest}"
           export EMBER_IMAGE="${EMBER_OVERRIDE:-niicloudoperation/rdm-ember-osf-web:latest}"
           export CAS_IMAGE="${CAS_OVERRIDE:-niicloudoperation/rdm-cas-overlay:latest}"
           export MFR_IMAGE="${MFR_OVERRIDE:-niicloudoperation/rdm-modular-file-renderer:latest}"
-          export WB_IMAGE="${WB_OVERRIDE:-niicloudoperation/rdm-waterbutler:latest}"
+          if [[ -n "$WB_IMAGE" ]]; then
+            export WB_IMAGE
+          else
+            export WB_IMAGE="${WB_OVERRIDE:-niicloudoperation/rdm-waterbutler:latest}"
+          fi
         else
           # Use workflow dispatch inputs or defaults
           export OSF_IMAGE="${{ github.event.inputs.osf_image || 'niicloudoperation/rdm-osf.io:latest' }}"
           export EMBER_IMAGE="${{ github.event.inputs.ember_image || 'niicloudoperation/rdm-ember-osf-web:latest' }}"
           export CAS_IMAGE="${{ github.event.inputs.cas_image || 'niicloudoperation/rdm-cas-overlay:latest' }}"
           export MFR_IMAGE="${{ github.event.inputs.mfr_image || 'niicloudoperation/rdm-modular-file-renderer:latest' }}"
-          export WB_IMAGE="${{ github.event.inputs.wb_image || 'niicloudoperation/rdm-waterbutler:latest' }}"
+          if [[ -n "$WB_IMAGE" ]]; then
+            export WB_IMAGE
+          else
+            export WB_IMAGE="${{ github.event.inputs.wb_image || 'niicloudoperation/rdm-waterbutler:latest' }}"
+          fi
         fi
-        
+
         source ${{ github.workspace }}/e2e-tests/.github/scripts/setup_rdm.sh
         create_docker_override
 
@@ -786,6 +859,72 @@ jobs:
         echo "S3COMPAT_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV
         echo "S3COMPAT_SERVICE_NAME=${MINIO_SERVICE_NAME}" >> $GITHUB_ENV
 
+    - name: Initialize MinIO for S3CompatSigV4 tests
+      if: env.MINIO_ENABLED == 'true' && (env.S3COMPATSIGV4_ENABLED == 'true' || env.S3COMPATSIGV4_INST_ENABLED == 'true')
+      working-directory: e2e-tests
+      run: |
+        set -eu
+
+        key_part() {
+          LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 18
+        }
+
+        secret_part() {
+          LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 40
+        }
+
+        BUCKET_PREFIX="ci-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-sigv4"
+
+        if [ "${S3COMPATSIGV4_ENABLED}" = "true" ]; then
+          ACCESS_KEY_1="ci$(key_part)"
+          SECRET_KEY_1="$(secret_part)"
+          ACCESS_KEY_2="ci$(key_part)"
+          SECRET_KEY_2="$(secret_part)"
+
+          BUCKET_NAME_1="${BUCKET_PREFIX}-a"
+          BUCKET_NAME_2="${BUCKET_PREFIX}-b"
+
+          .github/scripts/setup_minio_buckets.sh \
+            ../RDM-osf.io \
+            "${MINIO_ALIAS}" \
+            "${MINIO_ENDPOINT}" \
+            "${MINIO_ROOT_USER}" \
+            "${MINIO_ROOT_PASSWORD}" \
+            "${ACCESS_KEY_1}" "${SECRET_KEY_1}" "${BUCKET_NAME_1}" \
+            "${ACCESS_KEY_2}" "${SECRET_KEY_2}" "${BUCKET_NAME_2}"
+
+          echo "S3COMPATSIGV4_ACCESS_KEY_1=${ACCESS_KEY_1}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_SECRET_KEY_1=${SECRET_KEY_1}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_ACCESS_KEY_2=${ACCESS_KEY_2}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_SECRET_KEY_2=${SECRET_KEY_2}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_BUCKET_NAME_1=${BUCKET_NAME_1}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_BUCKET_NAME_2=${BUCKET_NAME_2}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_SERVICE_NAME=${MINIO_SERVICE_NAME}" >> $GITHUB_ENV
+        fi
+
+        if [ "${S3COMPATSIGV4_INST_ENABLED}" = "true" ]; then
+          INST_ACCESS_KEY="ci$(key_part)"
+          INST_SECRET_KEY="$(secret_part)"
+          INST_BUCKET="${BUCKET_PREFIX}-inst"
+          # Second bucket is required by setup_minio_buckets.sh but not used for inst storage
+          INST_BUCKET_DUMMY="${BUCKET_PREFIX}-inst2"
+
+          .github/scripts/setup_minio_buckets.sh \
+            ../RDM-osf.io \
+            "${MINIO_ALIAS}" \
+            "${MINIO_ENDPOINT}" \
+            "${MINIO_ROOT_USER}" \
+            "${MINIO_ROOT_PASSWORD}" \
+            "${INST_ACCESS_KEY}" "${INST_SECRET_KEY}" "${INST_BUCKET}" \
+            "${INST_ACCESS_KEY}" "${INST_SECRET_KEY}" "${INST_BUCKET_DUMMY}"
+
+          echo "S3COMPATSIGV4_INST_ACCESS_KEY=${INST_ACCESS_KEY}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_INST_SECRET_KEY=${INST_SECRET_KEY}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_INST_BUCKET=${INST_BUCKET}" >> $GITHUB_ENV
+          echo "S3COMPATSIGV4_INST_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV
+        fi
+
     - name: Start MinIO trace
       if: env.MINIO_ENABLED == 'true'
       working-directory: RDM-osf.io
@@ -1007,6 +1146,12 @@ jobs:
         if [ "${FLOWABLE_ENABLED}" = "true" ]; then
           args+=(--flowable)
         fi
+        if [ "${S3COMPATSIGV4_ENABLED}" = "true" ]; then
+          args+=(--s3compatsigv4)
+        fi
+        if [ "${S3COMPATSIGV4_INST_ENABLED}" = "true" ]; then
+          args+=(--s3compatsigv4-inst)
+        fi
 
         .github/scripts/generate_ci_config.sh "${args[@]}"
         rm base_ci.config.yaml