name: CI on: push: branches: - master - hotfix/** - release/** pull_request: branches: - master - hotfix/** - release/** permissions: id-token: write contents: read jobs: codemetrics: runs-on: ubuntu-latest name: "Log Code Metrics" if: github.ref == 'refs/heads/master' steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: node utils/codeMetrics.js env: CODE_METRICS_APP_ID: ${{ secrets.CODE_METRICS_APP_ID }} compile: runs-on: ubuntu-latest name: "Compile TypeScript" steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: npm run compile - run: npm run compile:strict format: runs-on: ubuntu-latest name: "Check Format" steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: npm run format:check lint: runs-on: ubuntu-latest name: "Lint" steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: npm run lint unittest: runs-on: ubuntu-latest name: "Unit Tests" steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: npm run test build: runs-on: ubuntu-latest name: "Build" steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: npm run build:contracts - name: Restore Build Cache uses: actions/cache@v4 with: path: .cache key: ${{ runner.os }}-build-cache - run: npm run pack:prod env: NODE_OPTIONS: "--max-old-space-size=4096" - run: cp -r ./Contracts ./dist/contracts - run: cp -r ./configs ./dist/configs - uses: actions/upload-artifact@v4 with: name: dist path: dist/ - name: "Az CLI login" uses: azure/login@v1 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.PREVIEW_SUBSCRIPTION_ID }} - name: Upload build to preview blob storage run: az storage blob upload-batch -d '$web' -s 'dist' --account-name ${{ secrets.PREVIEW_STORAGE_ACCOUNT_NAME }} --destination-path "${{github.event.pull_request.head.sha || github.sha}}" --auth-mode login --overwrite true - name: Upload preview config to blob storage run: az storage blob upload -c '$web' -f ./preview/config.json --account-name ${{ secrets.PREVIEW_STORAGE_ACCOUNT_NAME }} --name "${{github.event.pull_request.head.sha || github.sha}}/config.json" --auth-mode login --overwrite true nuget: name: Publish Nuget if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/') || contains(github.ref, 'release/') needs: [build] runs-on: ubuntu-latest env: NUGET_SOURCE: ${{ secrets.NUGET_SOURCE }} AZURE_DEVOPS_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} steps: - name: Download Dist Folder uses: actions/download-artifact@v4 with: name: dist - run: cp ./configs/prod.json config.json - run: dotnet nuget add source "$NUGET_SOURCE" --name "ADO" --username "jawelton@microsoft.com" --password "$AZURE_DEVOPS_PAT" --store-password-in-clear-text - run: dotnet pack DataExplorer.proj /p:PackageVersion="2.0.0-github-${GITHUB_SHA}" - run: dotnet nuget push "bin/Release/*.nupkg" --skip-duplicate --api-key Az --source="$NUGET_SOURCE" - run: dotnet nuget remove source "ADO" - uses: actions/upload-artifact@v4 name: Upload package to Artifacts with: name: prod-package path: "bin/Release/*.nupkg" nugetmpac: name: Publish Nuget MPAC if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/') || contains(github.ref, 'release/') needs: [build] runs-on: ubuntu-latest env: NUGET_SOURCE: ${{ secrets.NUGET_SOURCE }} AZURE_DEVOPS_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} steps: - name: Download Dist Folder uses: actions/download-artifact@v4 with: name: dist - run: cp ./configs/mpac.json config.json - run: sed -i 's/Azure.Cosmos.DB.Data.Explorer/Azure.Cosmos.DB.Data.Explorer.MPAC/g' DataExplorer.nuspec - run: dotnet nuget add source "$NUGET_SOURCE" --name "ADO" --username "jawelton@microsoft.com" --password "$AZURE_DEVOPS_PAT" --store-password-in-clear-text - run: dotnet pack DataExplorer.proj /p:PackageVersion="2.0.0-github-${GITHUB_SHA}" - run: dotnet nuget push "bin/Release/*.nupkg" --skip-duplicate --api-key Az --source="$NUGET_SOURCE" - run: dotnet nuget remove source "ADO" - uses: actions/upload-artifact@v4 name: Upload package to Artifacts with: name: mpac-package path: "bin/Release/*.nupkg" playwright-tests: name: "Run Playwright Tests (Shard ${{ matrix.shardIndex }} of ${{ matrix.shardTotal }})" runs-on: ubuntu-latest env: NODE_TLS_REJECT_UNAUTHORIZED: 0 AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} strategy: fail-fast: false matrix: shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] shardTotal: [20] steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x uses: actions/setup-node@v4 with: node-version: 18.x - run: npm ci - run: npx playwright install --with-deps - name: "Az CLI login" uses: Azure/login@v2 with: client-id: ${{ secrets.E2E_TESTS_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} # We can't use MSAL within playwright so we acquire tokens prior to running the tests - name: "Acquire RBAC tokens for test accounts" uses: azure/cli@v2 with: azcliversion: latest inlineScript: | SHARD_INDEX=${{ matrix.shardIndex }} echo PLAYWRIGHT_SHARD_INDEX=$SHARD_INDEX >> $GITHUB_ENV NOSQL_TESTACCOUNT_1_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-1.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_1_TOKEN" echo NOSQL_TESTACCOUNT_1_TOKEN=$NOSQL_TESTACCOUNT_1_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_2_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-2.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_2_TOKEN" echo NOSQL_TESTACCOUNT_2_TOKEN=$NOSQL_TESTACCOUNT_2_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_3_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-3.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_3_TOKEN" echo NOSQL_TESTACCOUNT_3_TOKEN=$NOSQL_TESTACCOUNT_3_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_4_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-4.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_4_TOKEN" echo NOSQL_TESTACCOUNT_4_TOKEN=$NOSQL_TESTACCOUNT_4_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_5_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-5.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_5_TOKEN" echo NOSQL_TESTACCOUNT_5_TOKEN=$NOSQL_TESTACCOUNT_5_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_6_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-6.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_6_TOKEN" echo NOSQL_TESTACCOUNT_6_TOKEN=$NOSQL_TESTACCOUNT_6_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_7_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-7.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_7_TOKEN" echo NOSQL_TESTACCOUNT_7_TOKEN=$NOSQL_TESTACCOUNT_7_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_8_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-8.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_8_TOKEN" echo NOSQL_TESTACCOUNT_8_TOKEN=$NOSQL_TESTACCOUNT_8_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_9_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-9.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_9_TOKEN" echo NOSQL_TESTACCOUNT_9_TOKEN=$NOSQL_TESTACCOUNT_9_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_10_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-10.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_10_TOKEN" echo NOSQL_TESTACCOUNT_10_TOKEN=$NOSQL_TESTACCOUNT_10_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_11_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-11.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_11_TOKEN" echo NOSQL_TESTACCOUNT_11_TOKEN=$NOSQL_TESTACCOUNT_11_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_12_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-12.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_12_TOKEN" echo NOSQL_TESTACCOUNT_12_TOKEN=$NOSQL_TESTACCOUNT_12_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_13_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-13.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_13_TOKEN" echo NOSQL_TESTACCOUNT_13_TOKEN=$NOSQL_TESTACCOUNT_13_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_14_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-14.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_14_TOKEN" echo NOSQL_TESTACCOUNT_14_TOKEN=$NOSQL_TESTACCOUNT_14_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_15_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-15.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_15_TOKEN" echo NOSQL_TESTACCOUNT_15_TOKEN=$NOSQL_TESTACCOUNT_15_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_16_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-16.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_16_TOKEN" echo NOSQL_TESTACCOUNT_16_TOKEN=$NOSQL_TESTACCOUNT_16_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_17_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-17.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_17_TOKEN" echo NOSQL_TESTACCOUNT_17_TOKEN=$NOSQL_TESTACCOUNT_17_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_18_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-18.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_18_TOKEN" echo NOSQL_TESTACCOUNT_18_TOKEN=$NOSQL_TESTACCOUNT_18_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_19_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-19.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_19_TOKEN" echo NOSQL_TESTACCOUNT_19_TOKEN=$NOSQL_TESTACCOUNT_19_TOKEN >> $GITHUB_ENV NOSQL_TESTACCOUNT_20_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-20.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_TESTACCOUNT_20_TOKEN" echo NOSQL_TESTACCOUNT_20_TOKEN=$NOSQL_TESTACCOUNT_20_TOKEN >> $GITHUB_ENV NOSQL_READONLY_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-readonly.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_READONLY_TESTACCOUNT_TOKEN" echo NOSQL_READONLY_TESTACCOUNT_TOKEN=$NOSQL_READONLY_TESTACCOUNT_TOKEN >> $GITHUB_ENV NOSQL_CONTAINERCOPY_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-sql-containercopyonly.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$NOSQL_CONTAINERCOPY_TESTACCOUNT_TOKEN" echo NOSQL_CONTAINERCOPY_TESTACCOUNT_TOKEN=$NOSQL_CONTAINERCOPY_TESTACCOUNT_TOKEN >> $GITHUB_ENV TABLE_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-tables.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$TABLE_TESTACCOUNT_TOKEN" echo TABLE_TESTACCOUNT_TOKEN=$TABLE_TESTACCOUNT_TOKEN >> $GITHUB_ENV GREMLIN_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-gremlin.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$GREMLIN_TESTACCOUNT_TOKEN" echo GREMLIN_TESTACCOUNT_TOKEN=$GREMLIN_TESTACCOUNT_TOKEN >> $GITHUB_ENV CASSANDRA_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-cassandra.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$CASSANDRA_TESTACCOUNT_TOKEN" echo CASSANDRA_TESTACCOUNT_TOKEN=$CASSANDRA_TESTACCOUNT_TOKEN >> $GITHUB_ENV MONGO_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-mongo.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$MONGO_TESTACCOUNT_TOKEN" echo MONGO_TESTACCOUNT_TOKEN=$MONGO_TESTACCOUNT_TOKEN >> $GITHUB_ENV MONGO32_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-mongo32.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$MONGO32_TESTACCOUNT_TOKEN" echo MONGO32_TESTACCOUNT_TOKEN=$MONGO32_TESTACCOUNT_TOKEN >> $GITHUB_ENV MONGO_READONLY_TESTACCOUNT_TOKEN=$(az account get-access-token --scope "https://github-e2etests-mongo-readonly.documents.azure.com/.default" -o tsv --query accessToken) echo "::add-mask::$MONGO_READONLY_TESTACCOUNT_TOKEN" echo MONGO_READONLY_TESTACCOUNT_TOKEN=$MONGO_READONLY_TESTACCOUNT_TOKEN >> $GITHUB_ENV - name: List test files for shard ${{ matrix['shardIndex'] }} of ${{ matrix['shardTotal']}} run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --list - name: Run test shard ${{ matrix['shardIndex'] }} of ${{ matrix['shardTotal']}} run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers=3 - name: "Re-auth for upload (refresh OIDC token)" if: ${{ !cancelled() }} uses: Azure/login@v2 with: client-id: ${{ secrets.E2E_TESTS_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Upload shard blob-report to Azure Storage if: ${{ !cancelled() }} env: KEY: ${{ github.run_id }}-${{ github.run_attempt }} SHARD: ${{ matrix.shardIndex }} run: | az storage blob upload-batch \ --account-name ${{ secrets.PREVIEW_STORAGE_ACCOUNT_NAME }} \ --auth-mode login \ -d playwright-reports \ -s blob-report \ --destination-path "${KEY}/shards/shard-${SHARD}" \ --overwrite true merge-playwright-reports: name: "Merge Playwright Reports" # Merge reports after playwright-tests, even if some shards have failed if: ${{ !cancelled() }} needs: [playwright-tests] runs-on: ubuntu-latest permissions: contents: read pull-requests: write id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 18 - name: Install dependencies run: npm ci - name: "Az CLI login" uses: Azure/login@v2 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.PREVIEW_SUBSCRIPTION_ID }} - name: Download all shard reports from Azure Storage env: KEY: ${{ github.run_id }}-${{ github.run_attempt }} run: | mkdir -p all-blob-reports az storage blob download-batch \ --account-name ${{ secrets.PREVIEW_STORAGE_ACCOUNT_NAME }} \ --auth-mode login \ -s playwright-reports \ --pattern "${KEY}/shards/*" \ -d ./all-blob-reports find ./all-blob-reports -type f -name "*.zip" -exec mv -t ./all-blob-reports {} + find ./all-blob-reports -type d -empty -delete ls -la ./all-blob-reports - name: Merge into HTML Report run: npx playwright merge-reports --reporter html ./all-blob-reports - name: Bundle merged report into a single zip run: | cd playwright-report zip -r ../report.zip . cd .. - name: Upload report.zip to Azure Storage env: KEY: ${{ github.run_id }}-${{ github.run_attempt }} run: | az storage blob upload \ --account-name ${{ secrets.PREVIEW_STORAGE_ACCOUNT_NAME }} \ --auth-mode login \ -c playwright-reports \ -n "${KEY}/report.zip" \ -f report.zip \ --overwrite - name: Clean up shard intermediates from Azure Storage if: ${{ always() }} env: KEY: ${{ github.run_id }}-${{ github.run_attempt }} run: | az storage blob delete-batch \ --account-name ${{ secrets.PREVIEW_STORAGE_ACCOUNT_NAME }} \ --auth-mode login \ -s playwright-reports \ --pattern "${KEY}/shards/*" - name: Comment Playwright results on PR if: ${{ !cancelled() && github.event_name == 'pull_request' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR: ${{ github.event.pull_request.number }} SA_ID: /subscriptions/${{ secrets.PREVIEW_SUBSCRIPTION_ID }}/resourceGroups/dataexplorer-preview/providers/Microsoft.Storage/storageAccounts/dataexplorerpreview KEY: ${{ github.run_id }}-${{ github.run_attempt }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | PLAYWRIGHT_JSON_OUTPUT_NAME=results.json npx playwright merge-reports --reporter json ./all-blob-reports read PASSED FAILED FLAKY SKIPPED DURATION < <(jq -r '[.stats.expected,.stats.unexpected,.stats.flaky,.stats.skipped,(.stats.duration/1000|floor)] | @tsv' results.json) BROKEN=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs" \ --jq '[.jobs[] | select(.name | startswith("Run Playwright Tests")) | select(.conclusion == "failure")] | length') if [ "$FAILED" -gt 0 ] || [ "$BROKEN" -gt 0 ]; then ICON="โŒ failed"; else ICON="โœ… passed"; fi SA_ENC=$(printf '%s' "$SA_ID" | jq -sRr @uri) CONTAINER_URL="https://portal.azure.com/#view/Microsoft_Azure_Storage/ContainerMenuBlade/~/overview/storageAccountId/${SA_ENC}/path/playwright-reports" NOTE="" [ "$BROKEN" -gt 0 ] && NOTE=" > โš ๏ธ $BROKEN shard(s) failed before tests ran (infra/auth issue). Stats below reflect only shards that executed." gh pr comment "$PR" --body "### Playwright tests $ICON | Passed | Failed | Flaky | Duration | | :---: | :---: | :---: | :---: | | $PASSED | $FAILED | $FLAKY | ${DURATION}s | ๐Ÿ“ **Report:** \`${KEY}/report.zip\` [Open container]($CONTAINER_URL) (Azure sign-in required) โ†’ click into \`${KEY}\` folder โ†’ click \`report.zip\` โ†’ **Download** โ†’ unzip โ†’ open \`index.html\` ยท [Workflow run]($RUN_URL)$NOTE"