github-azure-pipeline.yml

# Azure Pipeline: Sync Microsoft Learn Docs to Agent Skills
#
# This pipeline:
# 1. Checks out GitHub repo (contains products/, skills/)
# 2. Checks out ADO repo (contains src/, config/)
# 3. Runs docs2skills to crawl and classify docs
# 4. Pushes results back to GitHub repo

trigger: none

parameters:
  - name: fullSync
    displayName: 'Full Sync (use --full flag)'
    type: boolean
    default: false
  - name: targetBranch
    displayName: 'Target GitHub Branch'
    type: string
    default: 'main'
  - name: docs2SkillsADOBranch
    displayName: 'Docs2Skills ADO Branch'
    type: string
    default: 'main'

schedules:
  - cron: "0 2 * * 0"  # Every Sunday at 2:00 AM UTC
    displayName: Run every Sunday
    branches:
      include:
        - main
    always: true

variables:
  - group: Docs2Skills-Defaults
  - name: pythonVersion
    value: '3.12'

stages:
  - stage: __default
    jobs:
      - job: Job
        pool:
          vmImage: ubuntu-latest
        timeoutInMinutes: 240

        steps:
          # 0. Check required variables
          - task: Bash@3
            displayName: 'Check required variables'
            inputs:
              targetType: inline
              script: |
                set -e
                missing_vars=""
                if [ -z "$(azureOpenAIDeployment)" ]; then
                  echo "ERROR: azureOpenAIDeployment is not set or empty."
                  missing_vars="1"
                fi
                if [ -z "$(azureOpenAIEndpoint)" ]; then
                  echo "ERROR: azureOpenAIEndpoint is not set or empty."
                  missing_vars="1"
                fi

          # 1. Checkout GitHub repo (self - contains products/, skills/)
          - task: Checkout@1
            displayName: 'Checkout GitHub repo'
            inputs:
              repository: self
              persistCredentials: true
 
          # 2. Checkout ADO repo (source code)
          - task: Checkout@1
            displayName: 'Checkout ADO repo (code)'
            inputs:
              repository: git://Engineering/Learn.Docs2AgentSkills@refs/heads/${{ parameters.docs2SkillsADOBranch }}

          # 3. Check if current branch is behind target branch
          - task: Bash@3
            displayName: 'Check behind target branch'
            inputs:
              targetType: inline
              script: |
                set -e
                GITHUB_REPO=$(basename $(Build.Repository.Uri) .git)
                cd $(Build.SourcesDirectory)/$GITHUB_REPO

                TARGET_BRANCH="${{ parameters.targetBranch }}"
                echo "Fetching target branch: $TARGET_BRANCH"
                git fetch origin $TARGET_BRANCH || true

                if ! git show-ref --verify --quiet refs/remotes/origin/$TARGET_BRANCH; then
                  echo "Target branch origin/$TARGET_BRANCH does not exist yet. Skipping behind/ahead check."
                  exit 0
                fi

                BEHIND_COUNT=$(git rev-list --count HEAD..origin/$TARGET_BRANCH)
                AHEAD_COUNT=$(git rev-list --count origin/$TARGET_BRANCH..HEAD)

                echo "Branch status vs origin/$TARGET_BRANCH: ahead=$AHEAD_COUNT, behind=$BEHIND_COUNT"

                if [ "$BEHIND_COUNT" -gt 0 ]; then
                  echo "ERROR: Current branch is behind origin/$TARGET_BRANCH by $BEHIND_COUNT commit(s)."
                  echo "Please rebase/merge latest target branch before running sync."
                  exit 1
                fi

          # 4. Setup Python
          - task: UsePythonVersion@0
            displayName: 'Setup Python $(pythonVersion)'
            inputs:
              versionSpec: '$(pythonVersion)'

          # 5. Install dependencies
          - task: Bash@3
            displayName: 'Install dependencies'
            inputs:
              targetType: inline
              script: |
                set -e
                cd $(Build.SourcesDirectory)/Learn.Docs2AgentSkills
                pip install --upgrade pip
                pip install -r requirements.txt

          # 6. Run docs2skills with Azure identity
          - task: AzureCLI@2
            displayName: 'Run docs2skills'
            inputs:
              azureSubscription: $(azureSubscription)
              scriptType: bash
              scriptLocation: inlineScript
              inlineScript: |
                set -e

                # Define paths
                ADO_REPO="$(Build.SourcesDirectory)/Learn.Docs2AgentSkills"
                GITHUB_REPO="$(Build.SourcesDirectory)/$(basename $(Build.Repository.Uri) .git)"
                PRODUCTS_DIR="$GITHUB_REPO/products"
                SKILLS_DIR="$GITHUB_REPO/skills"

                cd $ADO_REPO

                echo "========================================"
                echo "Starting docs sync"
                echo "========================================"
                echo "ADO repo (code): $ADO_REPO"
                echo "GitHub repo (data): $GITHUB_REPO"
                echo "Products dir: $PRODUCTS_DIR"
                echo "Skills dir: $SKILLS_DIR"
                echo "Azure OpenAI: $(azureOpenAIEndpoint)"
                echo "Deployment: $(azureOpenAIDeployment)"
                echo "Full Sync: ${{ parameters.fullSync }}"
                echo "Target Branch: ${{ parameters.targetBranch }}"
                echo "========================================"

                FULL_FLAG=""
                if [ "${{ parameters.fullSync }}" = "True" ]; then
                  FULL_FLAG="--full"
                  echo "Running with --full flag"
                    # check before delete
                    if [ -n "$PRODUCTS_DIR" ] && [ -d "$PRODUCTS_DIR" ]; then
                      rm -rf "$PRODUCTS_DIR"
                      mkdir -p "$PRODUCTS_DIR"
                    fi
                  
                    if [ -n "$SKILLS_DIR" ] && [ -d "$SKILLS_DIR" ]; then
                      rm -rf "$SKILLS_DIR"
                      mkdir -p "$SKILLS_DIR"
                    fi
                fi

                echo "========================================"
                echo "Processing Well-Architected Framework"
                echo "========================================"
                python -m src.docs2skills \
                  --config-type waf \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR

                echo "========================================"
                echo "Processing Cloud Adoption Framework"
                echo "========================================"
                python -m src.docs2skills \
                  --config-type caf \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR

                echo "========================================"
                echo "Processing Azure Architecture Center"
                echo "========================================"
                python -m src.docs2skills \
                  --config-type aac \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR

                echo "========================================"
                echo "Processing Azure Other Products"
                echo "========================================"
                python -m src.docs2skills \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR


                echo "========================================"
                echo "Sync completed"
                echo "========================================"
            env:
              AZURE_OPENAI_ENDPOINT: $(azureOpenAIEndpoint)
              AZURE_OPENAI_CHAT_DEPLOYMENT: $(azureOpenAIDeployment)



          # # 6.5. [TEST ONLY] Make a dummy change to trigger PR
          # - task: Bash@3
          #   displayName: '[TEST] Make dummy change'
          #   inputs:
          #     targetType: inline
          #     script: |
          #       set -e
          #       GITHUB_REPO=$(basename $(Build.Repository.Uri) .git)
          #       cd $(Build.SourcesDirectory)/$GITHUB_REPO
          #       echo "test-$(date +%s)" >> products/test-dummy.txt

          # 7. Push to GitHub repo and create PR
          - task: Bash@3
            displayName: 'Push to GitHub and create PR'
            inputs:
              targetType: inline
              script: |
                set -e
                GITHUB_REPO=$(basename $(Build.Repository.Uri) .git)
                cd $(Build.SourcesDirectory)/$GITHUB_REPO

                git config user.email "docs2skills-pipeline@microsoft.com"
                git config user.name "Docs2Skills Pipeline"

                CHANGED_FILES=$(git status --porcelain products/ skills/ | wc -l)

                if [ "$CHANGED_FILES" -gt 0 ]; then
                  echo "Changed files: $CHANGED_FILES"
                  git status --short products/ skills/

                  git add products/ skills/
                  TIMESTAMP=$(date +"%Y-%m-%d %H:%M")
                  git commit -m "$TIMESTAMP - Generate skills from docs (code: ADO/${{ parameters.docs2SkillsADOBranch }}, target: GitHub/${{ parameters.targetBranch }})"
                  
                  TARGET_BRANCH="${{ parameters.targetBranch }}"
                  TIMESTAMP_BRANCH=$(date +"%Y%m%d-%H%M%S")
                  PR_BRANCH="docs-sync-$TIMESTAMP_BRANCH"
                  
                  # Push to PR branch
                  echo "Pushing changes to PR branch: $PR_BRANCH"
                  git push origin HEAD:refs/heads/$PR_BRANCH
                  
                  # Create PR targeting the target branch
                  echo "Creating PR from $PR_BRANCH to $TARGET_BRANCH"
                  REPO_URL=$(Build.Repository.Uri)
                  # Extract owner/repo from URL
                  REPO_PATH=$(echo $REPO_URL | sed 's|.*github.com/||' | sed 's|.git||')
                  # Reuse the token already stored by persistCredentials: true
                  EXTRAHEADER_KEY="http.$(Build.Repository.Uri).extraheader"
                  
                  export GH_TOKEN=$(git config --get "$EXTRAHEADER_KEY" | sed 's/.*: basic //' | base64 -d | sed 's/.*://')
                  [ -n "$GH_TOKEN" ] || { echo "ERROR: GH_TOKEN is empty"; exit 1; }
                  
                  gh pr create \
                    --title "Sync docs to agent skills - $TIMESTAMP" \
                    --body "Automated docs sync" \
                    --base "$TARGET_BRANCH" \
                    --head "$PR_BRANCH" \
                    --repo "$REPO_PATH"

                  echo "PR created successfully"
                else
                  echo "No changes detected in products/ or skills/ directories"
                fi