|
| 1 | +#!/bin/bash |
| 2 | +# Ingest Maven/Gradle dependencies into the dotnet-public-maven Azure Artifacts feed. |
| 3 | +# |
| 4 | +# WHY THIS IS NEEDED: |
| 5 | +# CI builds run under CFSClean network isolation which blocks direct access to |
| 6 | +# Maven Central (repo.maven.apache.org). All Maven dependencies are resolved |
| 7 | +# through the dotnet-public-maven Azure Artifacts feed instead. However, this |
| 8 | +# feed requires an authenticated request the FIRST time a package is pulled |
| 9 | +# from upstream Maven Central — after that, anyone can read it anonymously. |
| 10 | +# |
| 11 | +# The CI pipeline's credential provider plugin (com.microsoft.azure.artifacts. |
| 12 | +# credprovider) skips authentication in Azure Pipelines (TF_BUILD=True), so |
| 13 | +# new packages MUST be pre-ingested locally before CI can use them. |
| 14 | +# |
| 15 | +# WHEN TO RUN: |
| 16 | +# After adding or updating any Maven/Gradle dependency in |
| 17 | +# src/Core/AndroidNative/build.gradle or settings.gradle. |
| 18 | +# |
| 19 | +# HOW IT WORKS: |
| 20 | +# 1. Acquires an auth token via the .NET Azure Artifacts credential provider |
| 21 | +# 2. Pre-ingests platform-specific artifacts (e.g. aapt2) for all OS variants |
| 22 | +# (macOS/Linux/Windows) since Gradle only resolves the local OS classifier |
| 23 | +# 3. Runs the Gradle build with --refresh-dependencies to bypass local cache |
| 24 | +# and force actual downloads through the feed (which triggers ingestion) |
| 25 | +# 4. For packages that Gradle's credential provider can't reach (e.g. AGP's |
| 26 | +# internal detachedConfiguration scopes), falls back to curl with Bearer |
| 27 | +# token to force-ingest the specific package URLs |
| 28 | +# |
| 29 | +# COMMON PITFALL: |
| 30 | +# Running ./gradlew build without --refresh-dependencies may appear to succeed |
| 31 | +# but actually resolves from ~/.gradle/caches/ (local cache from prior builds |
| 32 | +# that used mavenCentral() directly). This does NOT ingest into the feed. |
| 33 | +# |
| 34 | +# Prerequisites: |
| 35 | +# - JDK 17+ |
| 36 | +# - .NET Azure Artifacts credential provider installed |
| 37 | +# (https://github.com/microsoft/artifacts-credprovider#installation) |
| 38 | +# |
| 39 | +# Usage: |
| 40 | +# ./eng/ingest-maven-deps.sh |
| 41 | + |
| 42 | +set -euo pipefail |
| 43 | + |
| 44 | +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" |
| 45 | +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" |
| 46 | +ANDROID_DIR="$REPO_ROOT/src/Core/AndroidNative" |
| 47 | +FEED_URL="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-maven/maven/v1" |
| 48 | +CRED_PROVIDER="$HOME/.nuget/plugins/netcore/CredentialProvider.Microsoft/CredentialProvider.Microsoft.dll" |
| 49 | + |
| 50 | +echo "=== Maven Dependency Ingestion for dotnet-public-maven ===" |
| 51 | +echo "" |
| 52 | + |
| 53 | +# Step 1: Get auth token |
| 54 | +echo "Acquiring auth token..." |
| 55 | +if [ ! -f "$CRED_PROVIDER" ]; then |
| 56 | + echo "ERROR: Azure Artifacts credential provider not found at $CRED_PROVIDER" |
| 57 | + echo "Install it from: https://github.com/microsoft/artifacts-credprovider#installation" |
| 58 | + exit 1 |
| 59 | +fi |
| 60 | + |
| 61 | +TOKEN=$(dotnet "$CRED_PROVIDER" -U "$FEED_URL" -F Json -N true -I true 2>/dev/null \ |
| 62 | + | python3 -c "import json,sys; print(json.load(sys.stdin).get('Password',''))" 2>/dev/null) |
| 63 | + |
| 64 | +if [ -z "$TOKEN" ]; then |
| 65 | + echo "ERROR: Failed to acquire auth token. Make sure you're signed in to Azure DevOps." |
| 66 | + exit 1 |
| 67 | +fi |
| 68 | +echo "Token acquired." |
| 69 | + |
| 70 | +# Step 2: Ingest platform-specific artifacts for all OS variants |
| 71 | +# Gradle only resolves the classifier for the current OS (e.g. aapt2-osx.jar on macOS). |
| 72 | +# CI builds on Windows/Linux need their variants pre-ingested too. |
| 73 | +echo "" |
| 74 | +echo "Step 1/3: Ingesting cross-platform artifacts..." |
| 75 | +AAPT2_VERSION="8.11.1-12782657" |
| 76 | +for classifier in osx linux windows; do |
| 77 | + for ext in jar pom; do |
| 78 | + url="$FEED_URL/com/android/tools/build/aapt2/$AAPT2_VERSION/aapt2-$AAPT2_VERSION-$classifier.$ext" |
| 79 | + code=$(curl -s -o /dev/null -w "%{http_code}" --oauth2-bearer "$TOKEN" "$url" 2>/dev/null) |
| 80 | + echo " aapt2-$AAPT2_VERSION-$classifier.$ext: $code" |
| 81 | + done |
| 82 | +done |
| 83 | + |
| 84 | +# Step 3: Run Gradle build with refresh to ingest via credential provider |
| 85 | +echo "" |
| 86 | +echo "Step 2/3: Running Gradle build with --refresh-dependencies..." |
| 87 | +cd "$ANDROID_DIR" |
| 88 | +if ! ./gradlew build --no-daemon --refresh-dependencies \ |
| 89 | + -Dazure.artifacts.credprovider.nonInteractive=true \ |
| 90 | + -Dazure.artifacts.credprovider.isRetry=true 2>&1 | tail -20; then |
| 91 | + echo "WARNING: Initial Gradle build failed (expected if packages need ingestion). Continuing..." |
| 92 | +fi |
| 93 | + |
| 94 | +# Step 4: Loop — build, find missing packages, curl-ingest them |
| 95 | +echo "" |
| 96 | +echo "Step 3/3: Ingesting any remaining packages via REST API..." |
| 97 | +for i in $(seq 1 30); do |
| 98 | + result=$(./gradlew build --no-daemon \ |
| 99 | + -Dazure.artifacts.credprovider.nonInteractive=true 2>&1 || true) |
| 100 | + |
| 101 | + if echo "$result" | grep -q "BUILD SUCCESSFUL"; then |
| 102 | + echo "All dependencies ingested successfully! ✅" |
| 103 | + exit 0 |
| 104 | + fi |
| 105 | + |
| 106 | + # Extract failed URLs and curl them with auth |
| 107 | + urls=$(echo "$result" | grep "Could not GET\|Could not HEAD" \ |
| 108 | + | sed "s/.*'\(https:[^']*\)'.*/\1/" | sort -u | grep "pkgs.dev.azure.com" || true) |
| 109 | + count=$(echo "$urls" | grep -c "https" 2>/dev/null || echo "0") |
| 110 | + |
| 111 | + if [ "$count" = "0" ]; then |
| 112 | + # No feed URLs failing — might be a different error |
| 113 | + echo "Build failed but not due to feed issues. Check build output." |
| 114 | + echo "$result" | grep -i "error" | grep -v "warning" | head -5 |
| 115 | + exit 1 |
| 116 | + fi |
| 117 | + |
| 118 | + echo " Run $i: ingesting $count packages..." |
| 119 | + echo "$urls" | while read url; do |
| 120 | + [ -z "$url" ] && continue |
| 121 | + code=$(curl -s -o /dev/null -w "%{http_code}" --oauth2-bearer "$TOKEN" "$url" 2>/dev/null) |
| 122 | + if [ "$code" = "200" ]; then |
| 123 | + echo " ✅ $(basename "$url")" |
| 124 | + else |
| 125 | + echo " ❌ ($code) $(basename "$url")" |
| 126 | + fi |
| 127 | + done |
| 128 | +done |
| 129 | + |
| 130 | +echo "WARNING: Reached max iterations. Some packages may still need ingestion." |
| 131 | +exit 1 |
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?v=4&size=40){kind=avatar}
0 commit comments