diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 26e01101693..d4adcefefad 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,72 +6,60 @@ # Area: Performance # @adityapatwardhan -# Area: Portability -# @JamesWTruher - # Area: Security -# @TravisEz13 @PaulHigin -src/System.Management.Automation/security/wldpNativeMethods.cs @TravisEz13 @PaulHigin - -# Area: Documentation -.github/ @joeyaiello @TravisEz13 +src/System.Management.Automation/security/wldpNativeMethods.cs @TravisEz13 @seeminglyscience -# Area: Test -# @JamesWTruher @TravisEz13 @adityapatwardhan - -# Area: Cmdlets Core -# @JamesWTruher @SteveL-MSFT @anmenaga +# Area: CI Build +.github/workflows @PowerShell/powershell-maintainers +.github/actions @PowerShell/powershell-maintainers # Now, areas that should have paths or filters, although we might not have them defined # According to the docs, order here must be by precedence of the filter, with later rules overwritting # but the feature seems to make taking a union of all the matching rules. # Area: Cmdlets Management -src/Microsoft.PowerShell.Commands.Management/ @daxian-dbw @adityapatwardhan +# src/Microsoft.PowerShell.Commands.Management/ @daxian-dbw @adityapatwardhan # Area: Utility Cmdlets -src/Microsoft.PowerShell.Commands.Utility/ @JamesWTruher @PaulHigin +# src/Microsoft.PowerShell.Commands.Utility/ # Area: Console -src/Microsoft.PowerShell.ConsoleHost/ @daxian-dbw @anmenaga @TylerLeonhardt - -# Area: Demos -demos/ @joeyaiello @SteveL-MSFT @HemantMahawar +# src/Microsoft.PowerShell.ConsoleHost/ @daxian-dbw # Area: DSC -src/System.Management.Automation/DscSupport @TravisEz13 @SteveL-MSFT +# src/System.Management.Automation/DscSupport @TravisEz13 @SteveL-MSFT # Area: Engine # src/System.Management.Automation/engine @daxian-dbw # Area: Debugging # Must be below engine to override -src/System.Management.Automation/engine/debugger/ @PaulHigin +# src/System.Management.Automation/engine/debugger/ # Area: Help -src/System.Management.Automation/help @adityapatwardhan +src/System.Management.Automation/help @adityapatwardhan @daxian-dbw # Area: Intellisense # @daxian-dbw # Area: Language -src/System.Management.Automation/engine/parser @daxian-dbw +src/System.Management.Automation/engine/parser @daxian-dbw @seeminglyscience # Area: Providers -src/System.Management.Automation/namespaces @anmenaga +# src/System.Management.Automation/namespaces # Area: Remoting -src/System.Management.Automation/engine/remoting @PaulHigin +src/System.Management.Automation/engine/remoting @daxian-dbw @TravisEz13 # Areas: Build # Must be last -*.config @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin -*.props @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin -*.yml @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin -*.csproj @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin -build.* @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin -tools/ @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin -docker/ @daxian-dbw @TravisEz13 @adityapatwardhan @anmenaga @PaulHigin +*.config @PowerShell/powershell-maintainers +*.props @PowerShell/powershell-maintainers +*.yml @PowerShell/powershell-maintainers +*.csproj @PowerShell/powershell-maintainers +build.* @PowerShell/powershell-maintainers +tools/ @PowerShell/powershell-maintainers +# docker/ @PowerShell/powershell-maintainers # Area: Compliance tools/terms @TravisEz13 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a3dc6fd5198..27089847987 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -14,8 +14,7 @@ - Use the present tense and imperative mood when describing your changes - [ ] [Summarized changes](https://github.com/PowerShell/PowerShell/blob/master/.github/CONTRIBUTING.md#pull-request---submission) - [ ] [Make sure all `.h`, `.cpp`, `.cs`, `.ps1` and `.psm1` files have the correct copyright header](https://github.com/PowerShell/PowerShell/blob/master/.github/CONTRIBUTING.md#pull-request---submission) -- [ ] This PR is ready to merge and is not [Work in Progress](https://github.com/PowerShell/PowerShell/blob/master/.github/CONTRIBUTING.md#pull-request---work-in-progress). - - If the PR is work in progress, please add the prefix `WIP:` or `[ WIP ]` to the beginning of the title (the `WIP` bot will keep its status check at `Pending` while the prefix is present) and remove the prefix when the PR is ready. +- [ ] This PR is ready to merge. If this PR is a work in progress, please open this as a [Draft Pull Request and mark it as Ready to Review when it is ready to merge](https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests). - **[Breaking changes](https://github.com/PowerShell/PowerShell/blob/master/.github/CONTRIBUTING.md#making-breaking-changes)** - [ ] None - **OR** @@ -25,21 +24,8 @@ - [ ] Not Applicable - **OR** - [ ] [Documentation needed](https://github.com/PowerShell/PowerShell/blob/master/.github/CONTRIBUTING.md#pull-request---submission) - - [ ] Issue filed: + - [ ] Issue filed: - **Testing - New and feature** - [ ] N/A or can only be tested interactively - **OR** - [ ] [Make sure you've added a new test if existing tests do not effectively test the code changed](https://github.com/PowerShell/PowerShell/blob/master/.github/CONTRIBUTING.md#before-submitting) -- **Tooling** - - [ ] I have considered the user experience from a tooling perspective and don't believe tooling will be impacted. - - **OR** - - [ ] I have considered the user experience from a tooling perspective and opened an issue in the relevant tool repository. This may include: - - [ ] Impact on [PowerShell Editor Services](https://github.com/PowerShell/PowerShellEditorServices) which is used in the [PowerShell extension](https://github.com/PowerShell/vscode-powershell) for VSCode - (which runs in a different PS Host). - - [ ] Issue filed: - - [ ] Impact on Completions (both in the console and in editors) - one of PowerShell's most powerful features. - - [ ] Issue filed: - - [ ] Impact on [PSScriptAnalyzer](https://github.com/PowerShell/PSScriptAnalyzer) (which provides linting & formatting in the editor extensions). - - [ ] Issue filed: - - [ ] Impact on [EditorSyntax](https://github.com/PowerShell/EditorSyntax) (which provides syntax highlighting with in VSCode, GitHub, and many other editors). - - [ ] Issue filed: diff --git a/.github/action-filters.yml b/.github/action-filters.yml new file mode 100644 index 00000000000..9a61bc1947b --- /dev/null +++ b/.github/action-filters.yml @@ -0,0 +1,23 @@ +github: &github + - .github/actions/** + - .github/workflows/**-ci.yml +tools: &tools + - tools/buildCommon/** + - tools/ci.psm1 +props: &props + - '**.props' +tests: &tests + - test/powershell/** + - test/tools/** + - test/xUnit/** +mainSource: &mainSource + - src/** +buildModule: &buildModule + - build.psm1 +source: + - *github + - *tools + - *props + - *buildModule + - *mainSource + - *tests diff --git a/.github/actions/build/ci/action.yml b/.github/actions/build/ci/action.yml new file mode 100644 index 00000000000..93adaf6b17a --- /dev/null +++ b/.github/actions/build/ci/action.yml @@ -0,0 +1,52 @@ +name: CI Build +description: 'Builds PowerShell' +runs: + using: composite + steps: + - name: Capture Environment + if: success() || failure() + run: 'Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose' + shell: pwsh + - name: Set Build Name for Non-PR + if: github.event_name != 'PullRequest' + run: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))" + shell: pwsh + - uses: actions/setup-dotnet@v4 + with: + global-json-file: ./global.json + - name: Bootstrap + if: success() + run: |- + Write-Verbose -Verbose "Running Bootstrap..." + Import-Module .\tools\ci.psm1 + Invoke-CIInstall -SkipUser + Write-Verbose -Verbose "Start Sync-PSTags" + Sync-PSTags -AddRemoteIfMissing + Write-Verbose -Verbose "End Sync-PSTags" + shell: pwsh + - name: Build + if: success() + run: |- + Write-Verbose -Verbose "Running Build..." + Import-Module .\tools\ci.psm1 + Invoke-CIBuild + shell: pwsh + - name: xUnit Tests + if: success() + continue-on-error: true + run: |- + Write-Verbose -Verbose "Running xUnit tests..." + Import-Module .\tools\ci.psm1 + Restore-PSOptions + Invoke-CIxUnit -SkipFailing + shell: pwsh + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: build + path: ${{ runner.workspace }}/build + - name: Upload xunit artifact + uses: actions/upload-artifact@v4 + with: + name: testResults-xunit + path: ${{ runner.workspace }}/xunit diff --git a/.github/actions/infrastructure/path-filters/action.yml b/.github/actions/infrastructure/path-filters/action.yml new file mode 100644 index 00000000000..78426bdff03 --- /dev/null +++ b/.github/actions/infrastructure/path-filters/action.yml @@ -0,0 +1,105 @@ +name: Path Filters +description: 'Path Filters' +inputs: + GITHUB_TOKEN: + description: 'GitHub token' + required: true +outputs: + source: + description: 'Source code changes (composite of all changes)' + value: ${{ steps.filter.outputs.source }} + githubChanged: + description: 'GitHub workflow changes' + value: ${{ steps.filter.outputs.githubChanged }} + toolsChanged: + description: 'Tools changes' + value: ${{ steps.filter.outputs.toolsChanged }} + propsChanged: + description: 'Props changes' + value: ${{ steps.filter.outputs.propsChanged }} + testsChanged: + description: 'Tests changes' + value: ${{ steps.filter.outputs.testsChanged }} + mainSourceChanged: + description: 'Main source code changes (any changes in src/)' + value: ${{ steps.filter.outputs.mainSourceChanged }} + buildModuleChanged: + description: 'Build module changes' + value: ${{ steps.filter.outputs.buildModuleChanged }} +runs: + using: composite + steps: + - name: Check if GitHubWorkflowChanges is present + id: filter + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ inputs.GITHUB_TOKEN }} + script: | + console.log(`Event Name: ${context.eventName}`); + + // Just say everything changed if this is not a PR + if (context.eventName !== 'pull_request') { + console.log('Not a pull request, setting all outputs to true'); + core.setOutput('toolsChanged', true); + core.setOutput('githubChanged', true); + core.setOutput('propsChanged', true); + core.setOutput('testsChanged', true); + core.setOutput('mainSourceChanged', true); + core.setOutput('buildModuleChanged', true); + core.setOutput('source', true); + return; + } + + console.log(`Getting files changed in PR #${context.issue.number}`); + + // Fetch the list of files changed in the PR + let files = []; + let page = 1; + let fetchedFiles; + do { + fetchedFiles = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100, + page: page++ + }); + files = files.concat(fetchedFiles.data); + } while (fetchedFiles.data.length > 0); + + const actionsChanged = files.some(file => file.filename.startsWith('.github/actions')); + const workflowsChanged = files.some(file => file.filename.startsWith('.github/workflows')); + const githubChanged = actionsChanged || workflowsChanged; + + const toolsCiPsm1Changed = files.some(file => file.filename.startsWith('tools/ci.psm1')); + const toolsBuildCommonChanged = files.some(file => file.filename.startsWith('tools/buildCommon/')); + const toolsChanged = toolsCiPsm1Changed || toolsBuildCommonChanged; + + const propsChanged = files.some(file => file.filename.endsWith('.props')); + + const testsChanged = files.some(file => file.filename.startsWith('test/powershell/') || file.filename.startsWith('test/tools/') || file.filename.startsWith('test/xUnit/')); + + const mainSourceChanged = files.some(file => file.filename.startsWith('src/')); + + const buildModuleChanged = files.some(file => file.filename.startsWith('build.psm1')); + + const source = mainSourceChanged || toolsChanged || githubChanged || propsChanged || testsChanged; + + core.setOutput('toolsChanged', toolsChanged); + core.setOutput('githubChanged', githubChanged); + core.setOutput('propsChanged', propsChanged); + core.setOutput('testsChanged', testsChanged); + core.setOutput('mainSourceChanged', mainSourceChanged); + core.setOutput('buildModuleChanged', buildModuleChanged); + core.setOutput('source', source); + + - name: Capture outputs + run: | + Write-Verbose -Verbose "source: ${{ steps.filter.outputs.source }}" + Write-Verbose -Verbose "github: ${{ steps.filter.outputs.githubChanged }}" + Write-Verbose -Verbose "tools: ${{ steps.filter.outputs.toolsChanged }}" + Write-Verbose -Verbose "props: ${{ steps.filter.outputs.propsChanged }}" + Write-Verbose -Verbose "tests: ${{ steps.filter.outputs.testsChanged }}" + Write-Verbose -Verbose "mainSource: ${{ steps.filter.outputs.mainSourceChanged }}" + Write-Verbose -Verbose "buildModule: ${{ steps.filter.outputs.buildModuleChanged }}" + shell: pwsh diff --git a/.github/actions/test/linux-packaging/action.yml b/.github/actions/test/linux-packaging/action.yml new file mode 100644 index 00000000000..b4a9c3b55c0 --- /dev/null +++ b/.github/actions/test/linux-packaging/action.yml @@ -0,0 +1,95 @@ +name: linux_packaging +description: 'Test very basic Linux packaging' + +# This isn't working yet +# It fails with + +# ERROR: While executing gem ... (Gem::FilePermissionError) +# You don't have write permissions for the /var/lib/gems/2.7.0 directory. +# WARNING: Installation of gem dotenv 2.8.1 failed! Must resolve manually. + +runs: + using: composite + steps: + - name: Capture Environment + if: success() || failure() + run: 'Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose' + shell: pwsh + - name: Download Build Artifacts + uses: actions/download-artifact@v4 + with: + path: "${{ github.workspace }}" + - name: Capture Artifacts Directory + continue-on-error: true + run: Get-ChildItem "${{ github.workspace }}/build/*" -Recurse + shell: pwsh + + - name: Bootstrap + run: |- + Import-Module ./build.psm1 + Start-PSBootstrap -Scenario Package + shell: pwsh + - name: Capture Artifacts Directory + continue-on-error: true + run: Import-Module ./build.psm1 + shell: pwsh + - name: Extract Files + uses: actions/github-script@v7.0.0 + env: + DESTINATION_FOLDER: "${{ github.workspace }}/bins" + ARCHIVE_FILE_PATTERNS: "${{ github.workspace }}/build/build.zip" + with: + script: |- + const fs = require('fs').promises + const path = require('path') + const target = path.resolve(process.env.DESTINATION_FOLDER) + const patterns = process.env.ARCHIVE_FILE_PATTERNS + const globber = await glob.create(patterns) + await io.mkdirP(path.dirname(target)) + for await (const file of globber.globGenerator()) { + if ((await fs.lstat(file)).isDirectory()) continue + await exec.exec(`7z x ${file} -o${target} -aoa`) + } + - name: Fix permissions + continue-on-error: true + run: |- + find "${{ github.workspace }}/bins" -type d -exec chmod +rwx {} \; + find "${{ github.workspace }}/bins" -type f -exec chmod +rw {} \; + shell: bash + - name: Capture Extracted Build ZIP + continue-on-error: true + run: Get-ChildItem "${{ github.workspace }}/bins/*" -Recurse -ErrorAction SilentlyContinue + shell: pwsh + - name: Packaging Tests + if: success() + run: |- + Import-Module ./tools/ci.psm1 + Restore-PSOptions -PSOptionsPath '${{ github.workspace }}/build/psoptions.json' + $options = (Get-PSOptions) + $rootPath = '${{ github.workspace }}/bins' + $originalRootPath = Split-Path -path $options.Output + $path = Join-Path -path $rootPath -ChildPath (split-path -leaf -path $originalRootPath) + $pwshPath = Join-Path -path $path -ChildPath 'pwsh' + chmod a+x $pwshPath + $options.Output = $pwshPath + Set-PSOptions $options + Invoke-CIFinish + shell: pwsh + - name: Upload packages + run: |- + Get-ChildItem "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}/*.deb" -Recurse | ForEach-Object { + $packagePath = $_.FullName + Write-Host "Uploading $packagePath" + Write-Host "##vso[artifact.upload containerfolder=deb;artifactname=deb]$packagePath" + } + Get-ChildItem "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}/*.rpm" -Recurse | ForEach-Object { + $packagePath = $_.FullName + Write-Host "Uploading $packagePath" + Write-Host "##vso[artifact.upload containerfolder=rpm;artifactname=rpm]$packagePath" + } + Get-ChildItem "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}/*.tar.gz" -Recurse | ForEach-Object { + $packagePath = $_.FullName + Write-Host "Uploading $packagePath" + Write-Host "##vso[artifact.upload containerfolder=rpm;artifactname=rpm]$packagePath" + } + shell: pwsh diff --git a/.github/actions/test/nix/action.yml b/.github/actions/test/nix/action.yml new file mode 100644 index 00000000000..b338c398340 --- /dev/null +++ b/.github/actions/test/nix/action.yml @@ -0,0 +1,112 @@ +name: nix_test +description: 'Test PowerShell on non-Windows platforms' + +inputs: + purpose: + required: false + default: '' + type: string + tagSet: + required: false + default: CI + type: string + ctrfFolder: + required: false + default: ctrf + type: string + +runs: + using: composite + steps: + - name: Capture Environment + if: success() || failure() + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Environment' + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + Write-LogGroupEnd -Title 'Environment' + shell: pwsh + + - name: Download Build Artifacts + uses: actions/download-artifact@v4 + with: + path: "${{ github.workspace }}" + + - name: Capture Artifacts Directory + continue-on-error: true + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Artifacts Directory' + Get-ChildItem "${{ github.workspace }}/build/*" -Recurse + Write-LogGroupEnd -Title 'Artifacts Directory' + shell: pwsh + + - uses: actions/setup-dotnet@v4 + with: + global-json-file: ./global.json + + - name: Bootstrap + shell: pwsh + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Bootstrap' + Import-Module ./tools/ci.psm1 + Invoke-CIInstall -SkipUser + Write-LogGroupEnd -Title 'Bootstrap' + + - name: Extract Files + uses: actions/github-script@v7.0.0 + env: + DESTINATION_FOLDER: "${{ github.workspace }}/bins" + ARCHIVE_FILE_PATTERNS: "${{ github.workspace }}/build/build.zip" + with: + script: |- + const fs = require('fs').promises + const path = require('path') + const target = path.resolve(process.env.DESTINATION_FOLDER) + const patterns = process.env.ARCHIVE_FILE_PATTERNS + const globber = await glob.create(patterns) + await io.mkdirP(path.dirname(target)) + for await (const file of globber.globGenerator()) { + if ((await fs.lstat(file)).isDirectory()) continue + await exec.exec(`7z x ${file} -o${target} -aoa`) + } + + - name: Fix permissions + continue-on-error: true + run: |- + find "${{ github.workspace }}/bins" -type d -exec chmod +rwx {} \; + find "${{ github.workspace }}/bins" -type f -exec chmod +rw {} \; + shell: bash + + - name: Capture Extracted Build ZIP + continue-on-error: true + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Extracted Build ZIP' + Get-ChildItem "${{ github.workspace }}/bins/*" -Recurse -ErrorAction SilentlyContinue + Write-LogGroupEnd -Title 'Extracted Build ZIP' + shell: pwsh + + - name: Test + if: success() + run: |- + Import-Module ./tools/ci.psm1 + Restore-PSOptions -PSOptionsPath '${{ github.workspace }}/build/psoptions.json' + $options = (Get-PSOptions) + $rootPath = '${{ github.workspace }}/bins' + $originalRootPath = Split-Path -path $options.Output + $path = Join-Path -path $rootPath -ChildPath (split-path -leaf -path $originalRootPath) + $pwshPath = Join-Path -path $path -ChildPath 'pwsh' + chmod a+x $pwshPath + $options.Output = $pwshPath + Set-PSOptions $options + Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -TitlePrefix '${{ inputs.buildName }}' -OutputFormat NUnitXml + shell: pwsh + + - name: Convert, Publish, and Upload Pester Test Results + uses: "./.github/actions/test/process-pester-results" + with: + name: "${{ inputs.purpose }}-${{ inputs.tagSet }}" + testResultsFolder: "${{ runner.workspace }}/testResults" + ctrfFolder: "${{ inputs.ctrfFolder }}" diff --git a/.github/actions/test/process-pester-results/action.yml b/.github/actions/test/process-pester-results/action.yml new file mode 100644 index 00000000000..27b94f6ebcb --- /dev/null +++ b/.github/actions/test/process-pester-results/action.yml @@ -0,0 +1,27 @@ +name: process-pester-test-results +description: 'Process Pester test results' + +inputs: + name: + required: true + default: '' + type: string + testResultsFolder: + required: false + default: "${{ runner.workspace }}/testResults" + type: string + +runs: + using: composite + steps: + - name: Log Summary + run: |- + & "$env:GITHUB_ACTION_PATH/process-pester-results.ps1" -Name '${{ inputs.name }}' -TestResultsFolder '${{ inputs.testResultsFolder }}' + shell: pwsh + + - name: Upload testResults artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-pester-${{ inputs.name }} + path: ${{ runner.workspace }}/testResults diff --git a/.github/actions/test/process-pester-results/process-pester-results.ps1 b/.github/actions/test/process-pester-results/process-pester-results.ps1 new file mode 100644 index 00000000000..523de3bebaa --- /dev/null +++ b/.github/actions/test/process-pester-results/process-pester-results.ps1 @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +param( + [parameter(Mandatory)] + [string]$Name, + [parameter(Mandatory)] + [string]$TestResultsFolder +) + +Import-Module "$PSScriptRoot/../../../../build.psm1" + +if (-not $env:GITHUB_STEP_SUMMARY) { + Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment." + exit 1 +} + +$testCaseCount = 0 +$testErrorCount = 0 +$testFailureCount = 0 +$testNotRunCount = 0 +$testInconclusiveCount = 0 +$testIgnoredCount = 0 +$testSkippedCount = 0 +$testInvalidCount = 0 + +Get-ChildItem -Path "${TestResultsFolder}/*.xml" -Recurse | ForEach-Object { + $results = [xml] (get-content $_.FullName) + + $testCaseCount += [int]$results.'test-results'.total + $testErrorCount += [int]$results.'test-results'.errors + $testFailureCount += [int]$results.'test-results'.failures + $testNotRunCount += [int]$results.'test-results'.'not-run' + $testInconclusiveCount += [int]$results.'test-results'.inconclusive + $testIgnoredCount += [int]$results.'test-results'.ignored + $testSkippedCount += [int]$results.'test-results'.skipped + $testInvalidCount += [int]$results.'test-results'.invalid +} + +@" + +# Summary of $Name + +- Total Tests: $testCaseCount +- Total Errors: $testErrorCount +- Total Failures: $testFailureCount +- Total Not Run: $testNotRunCount +- Total Inconclusive: $testInconclusiveCount +- Total Ignored: $testIgnoredCount +- Total Skipped: $testSkippedCount +- Total Invalid: $testInvalidCount + +"@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append + +Write-Log "Summary written to $ENV:GITHUB_STEP_SUMMARY" + +Write-LogGroupStart -Title 'Test Results' +Get-Content $ENV:GITHUB_STEP_SUMMARY +Write-LogGroupEnd -Title 'Test Results' + +if ($testErrorCount -gt 0 -or $testFailureCount -gt 0) { + Write-Error "There were $testErrorCount/$testFailureCount errors/failures in the test results." + exit 1 +} +if ($testCaseCount -eq 0) { + Write-Error "No test cases were run." + exit 1 +} diff --git a/.github/actions/test/verify_xunit/action.yml b/.github/actions/test/verify_xunit/action.yml new file mode 100644 index 00000000000..fccca27182f --- /dev/null +++ b/.github/actions/test/verify_xunit/action.yml @@ -0,0 +1,21 @@ +name: verify_xunit +description: 'Verify xUnit Results' + +runs: + using: composite + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + path: "${{ github.workspace }}" + - name: Capture artifacts directory + continue-on-error: true + run: dir "${{ github.workspace }}\testResults-xunit\*" -Recurse + shell: pwsh + - name: Test + if: success() + run: |- + Import-Module .\tools\ci.psm1 + $xUnitTestResultsFile = "${{ github.workspace }}\testResults-xunit\xUnitTestResults.xml" + Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile + shell: pwsh diff --git a/.github/actions/test/windows/action.yml b/.github/actions/test/windows/action.yml new file mode 100644 index 00000000000..734e30208f0 --- /dev/null +++ b/.github/actions/test/windows/action.yml @@ -0,0 +1,83 @@ +name: windows_test +description: 'Test PowerShell on Windows' + +inputs: + purpose: + required: false + default: '' + type: string + tagSet: + required: false + default: CI + type: string + ctrfFolder: + required: false + default: ctrf + type: string + +runs: + using: composite + steps: + - name: Capture Environment + if: success() || failure() + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Environment' + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + Write-LogGroupEnd -Title 'Environment' + shell: pwsh + + - name: Download Build Artifacts + uses: actions/download-artifact@v4 + with: + path: "${{ github.workspace }}" + + - name: Capture Artifacts Directory + continue-on-error: true + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Artifacts Directory' + Get-ChildItem "${{ github.workspace }}/build/*" -Recurse + Write-LogGroupEnd -Title 'Artifacts Directory' + shell: pwsh + + - uses: actions/setup-dotnet@v4 + with: + global-json-file: .\global.json + + - name: Bootstrap + shell: powershell + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Bootstrap' + Write-Host "Old Path:" + Write-Host $env:Path + $dotnetPath = Join-Path $env:SystemDrive 'Program Files\dotnet' + $paths = $env:Path -split ";" | Where-Object { -not $_.StartsWith($dotnetPath) } + $env:Path = $paths -join ";" + Write-Host "New Path:" + Write-Host $env:Path + # Bootstrap + Import-Module .\tools\ci.psm1 + Invoke-CIInstall + Write-LogGroupEnd -Title 'Bootstrap' + + - name: Test + if: success() + run: |- + Import-Module .\build.psm1 -force + Import-Module .\tools\ci.psm1 + Restore-PSOptions -PSOptionsPath '${{ github.workspace }}\build\psoptions.json' + $options = (Get-PSOptions) + $path = split-path -path $options.Output + $rootPath = split-Path -path $path + Expand-Archive -Path '${{ github.workspace }}\build\build.zip' -DestinationPath $rootPath -Force + Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -OutputFormat NUnitXml + shell: pwsh + + - name: Convert, Publish, and Upload Pester Test Results + uses: "./.github/actions/test/process-pester-results" + with: + name: "${{ inputs.purpose }}-${{ inputs.tagSet }}" + testResultsFolder: ${{ runner.workspace }}\testResults + ctrfFolder: "${{ inputs.ctrfFolder }}" diff --git a/.github/workflows/AssignPrs.yml b/.github/workflows/AssignPrs.yml index 419d704ce1d..a01c0bb0950 100644 --- a/.github/workflows/AssignPrs.yml +++ b/.github/workflows/AssignPrs.yml @@ -1,18 +1,20 @@ name: Auto Assign PR Maintainer on: - pull_request: + issues: types: [opened, edited] permissions: contents: read jobs: run: + if: github.repository_owner == 'PowerShell' runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: wow-actions/auto-assign@67fafa03df61d7e5f201734a2fa60d1ab111880d # v3.0.2 + if: github.event.issue.pull_request with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # using the `org/team_slug` or `/team_slug` syntax to add git team as reviewers diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 37a76edfeef..00000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [master] - pull_request: - # The branches below must be a subset of the branches above - branches: [master] - -defaults: - run: - shell: pwsh - -env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - -permissions: - contents: read - -jobs: - analyze: - permissions: - actions: read # for github/codeql-action/init to get workflow details - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/analyze to upload SARIF results - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['csharp'] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection - - steps: - - name: Checkout repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: '0' - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - - run: | - Get-ChildItem -Path env: - name: Capture Environment - - - run: | - Import-Module .\tools\ci.psm1 - Invoke-CIInstall -SkipUser - name: Bootstrap - - - run: | - Import-Module .\tools\ci.psm1 - Invoke-CIBuild - name: Build - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 diff --git a/.github/workflows/createReminders.yml b/.github/workflows/createReminders.yml index ef2c5fa1cce..0333b635d59 100644 --- a/.github/workflows/createReminders.yml +++ b/.github/workflows/createReminders.yml @@ -9,6 +9,8 @@ permissions: jobs: reminder: + if: github.repository_owner == 'PowerShell' + permissions: issues: write # for agrc/create-reminder-action to set reminders on issues pull-requests: write # for agrc/create-reminder-action to set reminders on PRs diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml new file mode 100644 index 00000000000..794ef64b213 --- /dev/null +++ b/.github/workflows/labels.yml @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: Verify PR Labels + +on: + pull_request: + types: [opened, reopened, edited, labeled, unlabeled, synchronize] + +permissions: + contents: read + pull-requests: read + +jobs: + verify-labels: + if: github.repository_owner == 'PowerShell' + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v2 + + - name: Verify PR has label starting with 'cl-' + id: verify-labels + uses: actions/github-script@v6 + with: + script: | + const labels = context.payload.pull_request.labels.map(label => label.name.toLowerCase()); + if (!labels.some(label => label.startsWith('cl-'))) { + core.setFailed("Every PR must have at least one label starting with 'cl-'."); + } diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml new file mode 100644 index 00000000000..a7523f430cf --- /dev/null +++ b/.github/workflows/linux-ci.yml @@ -0,0 +1,249 @@ +name: Linux-CI + +run-name: "${{ github.ref_name }} - ${{ github.run_number }}" + +on: + workflow_dispatch: + + push: + branches: + - master + - release/** + - github-mirror + paths: + - "**" + - "!.github/ISSUE_TEMPLATE/**" + - "!.dependabot/config.yml" + - "!.pipelines/**" + - "!test/perf/**" + pull_request: + branches: + - master + - release/** + - github-mirror +# Path filters for PRs need to go into the changes job + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ contains(github.ref, 'merge')}} + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + FORCE_FEATURE: 'False' + FORCE_PACKAGE: 'False' + NUGET_KEY: none + POWERSHELL_TELEMETRY_OPTOUT: 1 + __SuppressAnsiEscapeSequences: 1 + nugetMultiFeedWarnLevel: none + system_debug: 'false' +jobs: + changes: + if: startsWith(github.repository_owner, 'azure') || github.repository_owner == 'PowerShell' + name: Change Detection + runs-on: ubuntu-latest + # Required permissions + permissions: + pull-requests: read + contents: read + + # Set job outputs to values from filter step + outputs: + source: ${{ steps.filter.outputs.source }} + steps: + - name: checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Change Detection + id: filter + uses: "./.github/actions/infrastructure/path-filters" + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ci_build: + name: Build PowerShell + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.source == 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + + - name: Build + uses: "./.github/actions/build/ci" + linux_test_unelevated_ci: + name: Linux Unelevated CI + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Linux Unelevated CI + uses: "./.github/actions/test/nix" + with: + purpose: UnelevatedPesterTests + tagSet: CI + linux_test_elevated_ci: + name: Linux Elevated CI + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Linux Elevated CI + uses: "./.github/actions/test/nix" + with: + purpose: ElevatedPesterTests + tagSet: CI + linux_test_unelevated_others: + name: Linux Unelevated Others + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Linux Unelevated Others + uses: "./.github/actions/test/nix" + with: + purpose: UnelevatedPesterTests + tagSet: Others + linux_test_elevated_others: + name: Linux Elevated Others + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Linux Elevated Others + uses: "./.github/actions/test/nix" + with: + purpose: ElevatedPesterTests + tagSet: Others + verify_xunit: + name: Verify xUnit test results + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Verify xUnit test results + uses: "./.github/actions/test/verify_xunit" + + analyze: + permissions: + actions: read # for github/codeql-action/init to get workflow details + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/analyze to upload SARIF results + name: Analyze + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.source == 'true' }} + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['csharp'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: '0' + + - uses: actions/setup-dotnet@v4 + with: + global-json-file: ./global.json + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + - run: | + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + name: Capture Environment + shell: pwsh + + - run: | + Import-Module .\tools\ci.psm1 + Invoke-CIInstall -SkipUser + name: Bootstrap + shell: pwsh + + - run: | + Import-Module .\tools\ci.psm1 + Invoke-CIBuild + name: Build + shell: pwsh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 + + ready_to_merge: + name: Linux ready to merge + needs: + - verify_xunit + - linux_test_elevated_ci + - linux_test_elevated_others + - linux_test_unelevated_ci + - linux_test_unelevated_others + - analyze + if: always() + uses: PowerShell/compliance/.github/workflows/ready-to-merge.yml@v1.0.0 + with: + needs_context: ${{ toJson(needs) }} + # TODO: Enable this when we have a Linux packaging workflow + + # ERROR: While executing gem ... (Gem::FilePermissionError) + # You don't have write permissions for the /var/lib/gems/2.7.0 directory. + # WARNING: Installation of gem dotenv 2.8.1 failed! Must resolve manually. + + # linux_packaging: + # name: Attempt Linux Packaging + # needs: ci_build + # runs-on: ubuntu-20.04 + # steps: + # - name: checkout + # uses: actions/checkout@v4.1.0 + # with: + # fetch-depth: 1000 + # - name: Verify xUnit test results + # uses: "./.github/actions/test/linux-packaging" diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml new file mode 100644 index 00000000000..9184dc088f0 --- /dev/null +++ b/.github/workflows/macos-ci.yml @@ -0,0 +1,185 @@ +name: macOS-CI + +run-name: "${{ github.ref_name }} - ${{ github.run_number }}" + +on: + push: + branches: + - master + - release/** + - github-mirror + paths: + - "**" + - "!.github/ISSUE_TEMPLATE/**" + - "!.dependabot/config.yml" + - "!.pipelines/**" + - "!test/perf/**" + pull_request: + branches: + - master + - release/** + - github-mirror +# Path filters for PRs need to go into the changes job + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ contains(github.ref, 'merge')}} + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + FORCE_FEATURE: 'False' + FORCE_PACKAGE: 'False' + HOMEBREW_NO_ANALYTICS: 1 + NUGET_KEY: none + POWERSHELL_TELEMETRY_OPTOUT: 1 + __SuppressAnsiEscapeSequences: 1 + nugetMultiFeedWarnLevel: none + system_debug: 'false' + +jobs: + changes: + name: Change Detection + runs-on: ubuntu-latest + if: startsWith(github.repository_owner, 'azure') || github.repository_owner == 'PowerShell' + # Required permissions + permissions: + pull-requests: read + contents: read + + # Set job outputs to values from filter step + outputs: + source: ${{ steps.filter.outputs.source }} + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + + - name: Change Detection + id: filter + uses: "./.github/actions/infrastructure/path-filters" + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ci_build: + name: Build PowerShell + runs-on: macos-latest + needs: changes + if: ${{ needs.changes.outputs.source == 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Build + uses: "./.github/actions/build/ci" + macos_test_unelevated_ci: + name: macos Unelevated CI + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: macos-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: macOS Unelevated CI + uses: "./.github/actions/test/nix" + with: + purpose: UnelevatedPesterTests + tagSet: CI + macos_test_elevated_ci: + name: macOS Elevated CI + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: macos-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: macOS Elevated CI + uses: "./.github/actions/test/nix" + with: + purpose: ElevatedPesterTests + tagSet: CI + macos_test_unelevated_others: + name: macOS Unelevated Others + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: macos-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: macOS Unelevated Others + uses: "./.github/actions/test/nix" + with: + purpose: UnelevatedPesterTests + tagSet: Others + macos_test_elevated_others: + name: macOS Elevated Others + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: macos-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: macOS Elevated Others + uses: "./.github/actions/test/nix" + with: + purpose: ElevatedPesterTests + tagSet: Others + verify_xunit: + name: Verify xUnit test results + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Verify xUnit test results + uses: "./.github/actions/test/verify_xunit" + PackageMac-macos_packaging: + name: macOS packaging (bootstrap only) + needs: + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: + - macos-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + - name: Bootstrap packaging + if: success() || failure() + run: |- + import-module ./build.psm1 + start-psbootstrap -Scenario package + shell: pwsh + ready_to_merge: + name: macos ready to merge + needs: + - verify_xunit + - PackageMac-macos_packaging + - macos_test_elevated_ci + - macos_test_elevated_others + - macos_test_unelevated_ci + - macos_test_unelevated_others + if: always() + uses: PowerShell/compliance/.github/workflows/ready-to-merge.yml@v1.0.0 + with: + needs_context: ${{ toJson(needs) }} diff --git a/.github/workflows/markdownLink.yml b/.github/workflows/markdownLink.yml index baf668d83d1..85b9f51a742 100644 --- a/.github/workflows/markdownLink.yml +++ b/.github/workflows/markdownLink.yml @@ -11,6 +11,8 @@ permissions: jobs: markdown-link-check: runs-on: ubuntu-latest + if: github.repository_owner == 'PowerShell' + steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 @@ -31,13 +33,20 @@ jobs: # Full git history is needed to get a proper # list of changed files within `super-linter` fetch-depth: 0 + - name: Load super-linter configuration + # Use grep inverse matching to exclude eventual comments in the .env file + # because the GitHub Actions command to set environment variables doesn't + # support comments. + # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-environment-variable + run: grep -v '^#' tools/super-linter/config/super-linter.env >> "$GITHUB_ENV" - name: Lint Markdown uses: super-linter/super-linter@b4515bd4ad9d0aa4681960e053916ab991bdbe96 # v6.8.0 env: - VALIDATE_ALL_CODEBASE: false - DEFAULT_BRANCH: master - FILTER_REGEX_INCLUDE: .*\.md GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_EDITORCONFIG: false - VALIDATE_JSCPD: false - VALIDATE_CHECKOV: false + - name: Super-Linter correction instructions + if: failure() + uses: actions/github-script@v7.0.1 + with: + script: | + const message = "Super-Linter found issues in the changed files. Please check the logs for details. You can run the linter locally using the command: `./tools/super-lister/super-lister.ps1`."; + core.setFailed(message); diff --git a/.github/workflows/processReminders.yml b/.github/workflows/processReminders.yml index c6ac936343c..a2d5b4dbd93 100644 --- a/.github/workflows/processReminders.yml +++ b/.github/workflows/processReminders.yml @@ -10,6 +10,7 @@ permissions: jobs: reminder: + if: github.repository_owner == 'PowerShell' permissions: issues: write # for agrc/reminder-action to set reminders on issues pull-requests: write # for agrc/reminder-action to set reminders on PRs diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml deleted file mode 100644 index df5b4789fb3..00000000000 --- a/.github/workflows/rebase.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This cannot rebase workflow changes into a PR -# It also only works if the GITHUB_TOKEN has permission to push to the branch -# see: https://github.com/cirrus-actions/rebase/issues/12#issuecomment-632594995 -on: - issue_comment: - types: [created] -name: Automatic Rebase -permissions: - contents: read - -jobs: - rebase: - permissions: - contents: write # for cirrus-actions/rebase to push code to rebase - pull-requests: write # for actions/github-script to create PR comment - name: Rebase - if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: 0 - - name: Post rebase started comment to pull request - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - continue-on-error: true - with: - script: | - const backport_start_body = `Started rebase: https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; - await github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: backport_start_body - }); - - name: Automatic Rebase - uses: cirrus-actions/rebase@b87d48154a87a85666003575337e27b8cd65f691 # 1.8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 21430546a1f..90ac8e0b762 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -20,6 +20,7 @@ permissions: read-all jobs: analysis: name: Scorecard analysis + if: github.repository_owner == 'PowerShell' runs-on: ubuntu-latest permissions: # Needed to upload the results to code-scanning dashboard. diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml new file mode 100644 index 00000000000..1e955c78be6 --- /dev/null +++ b/.github/workflows/windows-ci.yml @@ -0,0 +1,169 @@ +name: Windows-CI +on: + workflow_dispatch: + push: + branches: + - master + - release/** + - github-mirror + paths: + - "**" + - "!.vsts-ci/misc-analysis.yml" + - "!.github/ISSUE_TEMPLATE/**" + - "!.dependabot/config.yml" + - "!test/perf/**" + - "!.pipelines/**" + pull_request: + branches: + - master + - release/** + - github-mirror + +# Path filters for PRs need to go into the changes job + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ contains(github.ref, 'merge')}} + +permissions: + contents: read + +run-name: "${{ github.ref_name }} - ${{ github.run_number }}" + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'" + NugetSecurityAnalysisWarningLevel: none + POWERSHELL_TELEMETRY_OPTOUT: 1 + __SuppressAnsiEscapeSequences: 1 + nugetMultiFeedWarnLevel: none +jobs: + changes: + name: Change Detection + runs-on: ubuntu-latest + if: startsWith(github.repository_owner, 'azure') || github.repository_owner == 'PowerShell' + # Required permissions + permissions: + pull-requests: read + contents: read + + # Set job outputs to values from filter step + outputs: + source: ${{ steps.filter.outputs.source }} + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + + - name: Change Detection + id: filter + uses: "./.github/actions/infrastructure/path-filters" + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ci_build: + name: Build PowerShell + needs: changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Build + uses: "./.github/actions/build/ci" + windows_test_unelevated_ci: + name: Windows Unelevated CI + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Windows Unelevated CI + uses: "./.github/actions/test/windows" + with: + purpose: UnelevatedPesterTests + tagSet: CI + windows_test_elevated_ci: + name: Windows Elevated CI + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Windows Elevated CI + uses: "./.github/actions/test/windows" + with: + purpose: ElevatedPesterTests + tagSet: CI + windows_test_unelevated_others: + name: Windows Unelevated Others + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Windows Unelevated Others + uses: "./.github/actions/test/windows" + with: + purpose: UnelevatedPesterTests + tagSet: Others + windows_test_elevated_others: + name: Windows Elevated Others + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Windows Elevated Others + uses: "./.github/actions/test/windows" + with: + purpose: ElevatedPesterTests + tagSet: Others + verify_xunit: + name: Verify xUnit test results + needs: + - ci_build + - changes + if: ${{ needs.changes.outputs.source == 'true' }} + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + with: + fetch-depth: 1000 + - name: Verify xUnit test results + uses: "./.github/actions/test/verify_xunit" + ready_to_merge: + name: windows ready to merge + needs: + - verify_xunit + - windows_test_elevated_ci + - windows_test_elevated_others + - windows_test_unelevated_ci + - windows_test_unelevated_others + if: always() + uses: PowerShell/compliance/.github/workflows/ready-to-merge.yml@v1.0.0 + with: + needs_context: ${{ toJson(needs) }} diff --git a/.gitignore b/.gitignore index cb12a297984..ccadde27182 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,10 @@ msbuild.binlog # Ignore gzip files in the manpage folder assets/manpage/*.gz + +# Ignore files and folders generated by some gh cli extensions +tmp/* +.env.local + +# Ignore CTRF report files +crtf/* diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/RolloutSpec.json b/.pipelines/EV2Specs/ServiceGroupRoot/RolloutSpec.json new file mode 100644 index 00000000000..9ed971068cc --- /dev/null +++ b/.pipelines/EV2Specs/ServiceGroupRoot/RolloutSpec.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json", + "contentVersion": "1.0.0.0", + "rolloutMetadata": { + "serviceModelPath": "ServiceModel.json", + "ScopeBindingsPath": "ScopeBindings.json", + "name": "OneBranch-Demo-Container-Deployment", + "rolloutType": "Major", + "buildSource": { + "parameters": { + "versionFile": "buildver.txt" + } + }, + "Notification": { + "Email": { + "To": "default" + } + } + }, + "orchestratedSteps": [ + { + "name": "UploadLinuxContainer", + "targetType": "ServiceResource", + "targetName": "LinuxContainerUpload", + "actions": ["Shell/Run"] + } + ] +} diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json b/.pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json new file mode 100644 index 00000000000..c3a98555867 --- /dev/null +++ b/.pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/scopeBindings.json", + "contentVersion": "0.0.0.1", + "scopeBindings": [ + { + "scopeTagName": "Global", + "bindings": [ + { + "find": "__SUBSCRIPTION_ID__", + "replaceWith": "$azureSubscriptionId()" + }, + { + "find": "__RESOURCE_GROUP__", + "replaceWith": "$azureResourceGroup()" + }, + { + "find": "__BUILD_VERSION__", + "replaceWith": "$buildVersion()" + } + ] + } + ] +} diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json new file mode 100644 index 00000000000..00555349c35 --- /dev/null +++ b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/serviceModel.json", + "contentVersion": "1.0.0.0", + "serviceMetadata": { + "serviceGroup": "OneBranch-PowerShellDocker", + "environment": "Test" + }, + "serviceResourceGroupDefinitions": [ + { + "name": "OneBranch-PowerShellDocker-RGDef", + "serviceResourceDefinitions": [ + { + "name": "OneBranch-PowerShellDocker.Shell-SRDef", + "composedOf": { + "extension": { + "shell": [ + { + "type": "Run", + "properties": { + "imageName": "adm-mariner-20-l", + "imageVersion": "v11" + } + } + ] + } + } + } + ] + } + ], + "serviceResourceGroups": [ + { + "azureResourceGroupName": "default", + "location": "West US 3", + "instanceOf": "OneBranch-PowerShellDocker-RGDef", + "azureSubscriptionId": "default", + "scopeTags": [ + { + "name": "Global" + } + ], + "serviceResources": [ + { + "Name": "LinuxContainerUpload", + "InstanceOf": "OneBranch-PowerShellDocker.Shell-SRDef", + "RolloutParametersPath": "UploadLinux.Rollout.json" + } + ] + } + ] +} diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 new file mode 100644 index 00000000000..25a5686b33e --- /dev/null +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -0,0 +1,393 @@ +<# +This function gets info from pmc's derived list of all repositories and from mapping.json (which contains info on just the repositories powershell publishes packages to, their package formats, etc) +to create a list of repositories PowerShell cares about along with repository Ids, repository full Urls and associated package that will be published to it. +#> +function Get-MappedRepositoryIds { + param( + [Parameter(Mandatory)] + [hashtable] + $Mapping, + + [Parameter(Mandatory)] + $RepoList, + + # LTS is not consider a package in this context. + # LTS is just another package name. + [Parameter(Mandatory)] + [ValidateSet('stable', 'preview')] + $Channel + ) + + $mappedReposUsedByPwsh = @() + foreach ($package in $Mapping.Packages) + { + Write-Verbose "package: $package" + $packageChannel = $package.channel + if (!$packageChannel) { + $packageChannel = 'all' + } + + Write-Verbose "package channel: $packageChannel" + if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel) + { + $repoIds = [System.Collections.Generic.List[string]]::new() + $packageFormat = $package.PackageFormat + Write-Verbose "package format: $packageFormat" -Verbose + $extension = [System.io.path]::GetExtension($packageFormat) + $packageType = $extension -replace '^\.' + + if ($package.distribution.count -gt 1) { + throw "Package $($package | out-string) has more than one Distribution." + } + + foreach ($distribution in $package.distribution) + { + $urlGlob = $package.url + switch ($packageType) + { + 'deb' { + $urlGlob = $urlGlob + '-apt' + } + 'rpm' { + $urlGlob = $urlGlob + '-yum' + } + default { + throw "Unknown package type: $packageType" + } + } + + Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose + $repos = $RepoList | Where-Object { $_.name -eq $urlGlob } + + if ($repos.id) { + Write-Verbose "Found repo id: $($repos.id)" -Verbose + $repoIds.AddRange(([string[]]$repos.id)) + } + else { + Write-Failure "Could not find repo for $urlGlob" + } + + if ($repoIds.Count -gt 0) { + $mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() }) + } + } + } + } + + Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)" + return $mappedReposUsedByPwsh +} + +<# +This function creates package objects for the packages to be published, +with the package name (ie package name format resolve with channel based PackageName and pwsh version), repoId, distribution and package path. +#> +function Get-PackageObjects() { + param( + [Parameter(Mandatory)] + [psobject[]] + $RepoObjects, + + [Parameter(Mandatory)] + [string] + $ReleaseVersion, + + [Parameter(Mandatory)] + [string[]] + $PackageName + ) + + $packages = @() + + foreach ($pkg in $RepoObjects) + { + if ($pkg.RepoId.count -gt 1) { + throw "Package $($pkg.name) has more than one repo id." + } + + if ($pkg.Distribution.count -gt 1) { + throw "Package $($pkg.name) has more than one Distribution." + } + + $pkgRepo = $pkg.RepoId | Select-Object -First 1 + $pkgDistribution = $pkg.Distribution | Select-Object -First 1 + + foreach ($name in $PackageName) { + $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) + + if ($pkgName.EndsWith('.rpm')) { + $pkgName = $pkgName.Replace($ReleaseVersion, $ReleaseVersion.Replace('-', '_')) + } + + $packagePath = "$pwshPackagesFolder/$pkgName" + $packagePathExists = Test-Path -Path $packagePath + if (!$packagePathExists) + { + throw "package path $packagePath does not exist" + } + + Write-Verbose "Creating package info object for package '$pkgName' for repo '$pkgRepo'" + $packages += @{ + PackagePath = $packagePath + PackageName = $pkgName + RepoId = $pkgRepo + Distribution = $pkgDistribution + } + + Write-Verbose -Verbose "package info obj: Name: $pkgName RepoId: $pkgRepo Distribution: $pkgDistribution PackagePath: $packagePath" + } + } + + Write-Verbose -Verbose "count of packages objects: $($packages.Length)" + return $packages +} + +<# +This function stages, uploads and publishes the powershell packages to their associated repositories in PMC. +#> +function Publish-PackageToPMC() { + param( + [Parameter(Mandatory)] + [pscustomobject[]] + $PackageObject, + + [Parameter(Mandatory)] + [string] + $ConfigPath, + + [Parameter(Mandatory)] + [bool] + $SkipPublish + ) + + # Don't fail outright when an error occurs, but instead pool them until + # after attempting to publish every package. That way we can choose to + # proceed for a partial failure. + $errorMessage = [System.Collections.Generic.List[string]]::new() + foreach ($finalPackage in $PackageObject) + { + Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose + $packagePath = $finalPackage.PackagePath + $pkgRepo = $finalPackage.RepoId + + $extension = [System.io.path]::GetExtension($packagePath) + $packageType = $extension -replace '^\.' + Write-Verbose "packageType: $packageType" -Verbose + + $packageListJson = pmc --config $ConfigPath package $packageType list --file $packagePath + $list = $packageListJson | ConvertFrom-Json + + $packageId = @() + if ($list.count -ne 0) + { + Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose + $packageId = $list.results.id | Select-Object -First 1 + } + else { + # PMC UPLOAD COMMAND + Write-Verbose -Verbose "Uploading package, config: '$ConfigPath' package: '$packagePath'" + $uploadResult = $null + try { + $uploadResult = pmc --config $ConfigPath package upload $packagePath --type $packageType + } + catch { + $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + continue + } + + $packageId = ($uploadResult | ConvertFrom-Json).id + } + + Write-Verbose "Got package ID: '$packageId'" -Verbose + $distribution = $finalPackage.Distribution | select-object -First 1 + Write-Verbose "distribution: $distribution" -Verbose + + if (!$SkipPublish) + { + Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose + + if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) + { + throw "Unsupported package type: $packageType" + return 1 + } + else { + # PMC UPDATE COMMAND + $rawUpdateResponse = $null + try { + if ($packageType -eq 'rpm') { + $rawUpdateResponse = pmc --config $ConfigPath repo package update $pkgRepo --add-packages $packageId + } elseif ($packageType -eq 'deb') { + $rawUpdateResponse = pmc --config $ConfigPath repo package update $pkgRepo $distribution --add-packages $packageId + } + } + catch { + $errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + continue + } + + $state = ($rawUpdateResponse | ConvertFrom-Json).state + Write-Verbose -Verbose "update response state: $state" + if ($state -ne 'completed') { + $errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse") + continue + } + } + + # PMC PUBLISH COMMAND + # The CLI outputs messages and JSON in the same stream, so we must sift through it for now + # This is planned to be fixed with a switch in a later release + Write-Verbose -Verbose ([pscustomobject]($package + @{ + PackageId = $packageId + })) + + # At this point, the changes are staged and will eventually be publish. + # Running publish, causes them to go live "immediately" + $rawPublishResponse = $null + try { + $rawPublishResponse = pmc --config $ConfigPath repo publish $pkgRepo + } + catch { + $errorMessage.Add("Invoking final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + continue + } + + $publishState = ($rawPublishResponse | ConvertFrom-Json).state + Write-Verbose -Verbose "publish response state: $publishState" + if ($publishState -ne 'completed') { + $errorMessage.Add("Final publishing of package $($finalPackage.PackageName) to $pkgRepo failed: $rawPublishResponse") + continue + } + } else { + Write-Verbose -Verbose "Skipping Uploading package --config-file '$ConfigPath' package add '$packagePath' --repoID '$pkgRepo'" + } + } + + if ($errorMessage) { + throw $errorMessage -join [Environment]::NewLine + } +} + +if ($null -eq $env:MAPPING_FILE) +{ + Write-Verbose -Verbose "MAPPING_FILE variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:PWSH_PACKAGES_TARGZIP) +{ + Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:PMC_METADATA) +{ + Write-Verbose -Verbose "PMC_METADATA variable didn't get passed correctly" + return 1 +} + +try { + Write-Verbose -Verbose "Downloading files" + Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json + Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz + Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json + + # create variables to those paths and test them + $mappingFilePath = Join-Path "/package/unarchive/" -ChildPath "mapping.json" + $mappingFilePathExists = Test-Path $mappingFilePath + if (!$mappingFilePathExists) + { + Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" + return 1 + } + + $packagesTarPath = Join-Path -Path "/package/unarchive/" -ChildPath "packages.tar.gz" + $packagesTarPathExists = Test-Path $packagesTarPath + if (!$packagesTarPathExists) + { + Write-Verbose -Verbose "packages.tar.gz expected at $packagesTarPath does not exist" + return 1 + } + + # Extract files from 'packages.tar.gz' + Write-Verbose -Verbose "---Extracting files from packages.tar.gz---" + $pwshPackagesFolder = Join-Path -Path "/package/unarchive/" -ChildPath "packages" + New-Item -Path $pwshPackagesFolder -ItemType Directory + tar -xzvf $packagesTarPath -C $pwshPackagesFolder --force-local + Get-ChildItem $pwshPackagesFolder -Recurse + + $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" + $metadataFilePathExists = Test-Path $metadataFilePath + if (!$metadataFilePathExists) + { + Write-Verbose -Verbose "pmcMetadata.json expected at $metadataFilePath does not exist" + return 1 + } + + # files in the extracted Run dir + $configPath = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml' + $configPathExists = Test-Path -Path $configPath + if (!$configPathExists) + { + Write-Verbose -Verbose "settings.toml expected at $configPath does not exist" + return 1 + } + + $pythonDlFolder = Join-Path '/package/unarchive/Run' -ChildPath 'python_dl' + $pyPathExists = Test-Path -Path $pythonDlFolder + if (!$pyPathExists) + { + Write-Verbose -Verbose "python_dl expected at $pythonDlFolder does not exist" + return 1 + } + + Write-Verbose -Verbose "Installing pmc-cli" + pip install --upgrade pip + pip --version --verbose + pip install /package/unarchive/Run/python_dl/*.whl + + # Get metadata + $channel = "" + $packageNames = @() + $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json + $releaseVersion = $metadataContent.ReleaseTag.TrimStart('v') + $skipPublish = $metadataContent.SkipPublish + $lts = $metadataContent.LTS + + if ($releaseVersion.Contains('-')) { + $channel = 'preview' + $packageNames = @('powershell-preview') + } + else { + $channel = 'stable' + $packageNames = @('powershell') + } + + if ($lts) { + $packageNames += @('powershell-lts') + } + + Write-Verbose -Verbose "---Getting repository list---" + $rawResponse = pmc --config $configPath repo list --limit 800 + $response = $rawResponse | ConvertFrom-Json + $limit = $($response.limit) + $count = $($response.count) + Write-Verbose -Verbose "'pmc repo list' limit is: $limit and count is: $count" + $repoList = $response.results + + Write-Verbose -Verbose "---Getting package info---" + + + Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose + $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable + $mappedReposUsedByPwsh = Get-MappedRepositoryIds -Mapping $mapping -RepoList $repoList -Channel $channel + $packageObjects = Get-PackageObjects -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion + Write-Verbose -Verbose "skip publish $skipPublish" + Publish-PackageToPMC -PackageObject $packageObjects -ConfigPath $configPath -SkipPublish $skipPublish +} +catch { + Write-Error -ErrorAction Stop $_.Exception.Message + return 1 +} + +return 0 diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json new file mode 100644 index 00000000000..d7c75c2e216 --- /dev/null +++ b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutParameters.json", + "contentVersion": "1.0.0.0", + "shellExtensions": [ + { + "name": "Run", + "type": "Run", + "properties": { + "maxExecutionTime": "PT2H" + }, + "package": { + "reference": { + "path": "Shell/Run.tar" + } + }, + "launch": { + "command": [ + "/bin/bash", + "-c", + "pwsh ./Run/Run.ps1" + ], + "environmentVariables": [ + { + "name": "MAPPING_FILE", + "reference": + { + "path": "Parameters\\mapping.json" + } + }, + { + "name": "PWSH_PACKAGES_TARGZIP", + "reference": + { + "path": "Parameters\\packages.tar.gz" + } + }, + { + "name": "PMC_METADATA", + "reference": + { + "path": "Parameters\\pmcMetadata.json" + } + } + ], + "identity": { + "type": "userAssigned", + "userAssignedIdentities": [ + "default" + ] + } + } + } + ] +} diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt b/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt new file mode 100644 index 00000000000..7dea76edb3d --- /dev/null +++ b/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt @@ -0,0 +1 @@ +1.0.1 diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index d478f351252..11215302e46 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -1,4 +1,4 @@ -name: UnifiedPackageBuild-$(Build.BuildId) +name: bins-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) trigger: none parameters: @@ -138,6 +138,8 @@ extends: value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - name: ob_signing_setup_enabled value: false + - name: ob_sdl_sbom_enabled + value: false steps: - checkout: self diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml index a39b4e866fc..30b9e415215 100644 --- a/.pipelines/PowerShell-Packages-Official.yml +++ b/.pipelines/PowerShell-Packages-Official.yml @@ -24,6 +24,8 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: Skip Signing type: string default: 'NO' + +name: pkgs-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) variables: - name: CDP_DEFINITION_BUILD_COUNT @@ -49,7 +51,7 @@ variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # Docker image which is used to build the project - name: LinuxContainerImage value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 - group: mscodehub-feed-read-general @@ -62,7 +64,7 @@ variables: resources: pipelines: - pipeline: CoOrdinatedBuildPipeline - source: 'PowerShell-Coordinated Packages-Official' + source: 'PowerShell-Coordinated Binaries-Official' trigger: branches: include: @@ -81,6 +83,9 @@ extends: cloudvault: enabled: false featureFlags: + WindowsHostVersion: + Version: 2022 + Network: KS3 linuxEsrpSigning: true globalSdl: disableLegacyManifest: true diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml new file mode 100644 index 00000000000..2d644c7a5dd --- /dev/null +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -0,0 +1,103 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: skipPublish + displayName: Skip PMC Publish + type: boolean + default: false + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + +name: ev2-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: REPOROOT + value: $(Build.SourcesDirectory) + - name: OUTPUTROOT + value: $(REPOROOT)\out + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\.config\tsaoptions.json + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + - group: PoolNames + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated Binaries-Official' + + - pipeline: PSPackagesOfficial + source: 'PowerShell-Packages-Official' + trigger: + branches: + include: + - master + - releases/* + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates + parameters: + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: Netlock + linuxEsrpSigning: true + cloudvault: + enabled: false + globalSdl: + disableLegacyManifest: true + # disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + asyncSdl: + enabled: true + tsaOptionsFile: .config/tsaoptions.json + tsa: + enabled: true + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + break: false # always break the build on binskim issues in addition to TSA upload + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + tsaOptionsFile: .config\tsaoptions.json + stages: + - template: /.pipelines/templates/release-prep-for-ev2.yml@self + parameters: + skipPublish: ${{ parameters.skipPublish }} + + - template: /.pipelines/templates/release-publish-pmc.yml@self diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 30c820cabcc..12b2c839c69 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -17,8 +17,8 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: Skip Signing type: string default: 'NO' - - name: SkipPMCPublish - displayName: Skip PMC Publish + - name: SkipPublish + displayName: Skip Publishing to GitHub and Nuget type: boolean default: false - name: SkipPSInfraInstallers @@ -26,6 +26,8 @@ parameters: # parameters are shown up in ADO UI in a build queue time type: boolean default: false +name: release-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) + variables: - name: CDP_DEFINITION_BUILD_COUNT value: $[counter('', 0)] @@ -56,6 +58,7 @@ variables: - name: ReleaseTagVar value: ${{ parameters.ReleaseTagVar }} - group: PoolNames + - group: MSIXSigningProfile resources: repositories: @@ -70,7 +73,7 @@ resources: pipelines: - pipeline: CoOrdinatedBuildPipeline - source: 'PowerShell-Coordinated Packages-Official' + source: 'PowerShell-Coordinated Binaries-Official' - pipeline: PSPackagesOfficial source: 'PowerShell-Packages-Official' @@ -114,10 +117,10 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - - stage: setReleaseTagAndUploadTools - displayName: 'Set Release Tag and Upload Tools' + - stage: setReleaseTagAndChangelog + displayName: 'Set Release Tag and Upload Changelog' jobs: - - template: /.pipelines/templates/release-SetTagAndTools.yml@self + - template: /.pipelines/templates/release-SetTagAndChangelog.yml@self - stage: msixbundle displayName: 'Create MSIX Bundle' @@ -202,8 +205,10 @@ extends: displayName: 'Validate Linux ARM64 Fxd Packages' jobtype: 'linux' artifactName: 'drop_linux_package_fxdependent' + # this is really an architecture independent package packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' arm64: 'yes' + enableCredScan: false - stage: validatePackages displayName: 'Validate Packages' @@ -276,19 +281,21 @@ extends: Update and merge the changelog for the release. This step is required for creating GitHub draft release. - - stage: PublishGitHubRelease - displayName: Publish GitHub Release - dependsOn: - - setReleaseTagAndUploadTools + - stage: PublishGitHubReleaseAndNuget + displayName: Publish GitHub and Nuget Release + dependsOn: + - setReleaseTagAndChangelog - UpdateChangeLog variables: ob_release_environment: Production jobs: - - template: /.pipelines/templates/release-githubtasks.yml@self + - template: /.pipelines/templates/release-githubNuget.yml@self + parameters: + skipPublish: ${{ parameters.SkipPublish }} - stage: PushGitTagAndMakeDraftPublic displayName: Push Git Tag and Make Draft Public - dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubReleaseAndNuget jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -315,41 +322,24 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - stage: PublishNuGet - displayName: Publish NuGet - dependsOn: - - setReleaseTagAndUploadTools - - PushGitTagAndMakeDraftPublic - variables: - ob_release_environment: Production - jobs: - - template: /.pipelines/templates/release-publish-nuget.yml@self - parameters: - skipPublish: true - - stage: PublishPMC displayName: Publish PMC dependsOn: PushGitTagAndMakeDraftPublic jobs: - - template: /.pipelines/templates/release-publish-pmc.yml@self + - template: /.pipelines/templates/approvalJob.yml@self parameters: - skipPublish: ${{ parameters.SkipPMCPublish }} + displayName: Publish to PMC + jobName: ReleaseToPMC + instructions: | + Run PowerShell-Release-Official-Azure.yml pipeline to publish to PMC - stage: ReleaseDocker dependsOn: PushGitTagAndMakeDraftPublic displayName: 'Docker Release' jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Start Docker Build - jobName: StartDockerBuild - instructions: | - Kick off Docker build - - template: /.pipelines/templates/approvalJob.yml@self parameters: displayName: Start Docker Release - dependsOnJob: StartDockerBuild jobName: StartDockerRelease instructions: | Kickoff docker release @@ -423,8 +413,7 @@ extends: - stage: ChangesToMaster displayName: Ensure changes are in GH master - dependsOn: - - PublishNuGet + dependsOn: - PublishPMC jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -434,17 +423,6 @@ extends: instructions: | Make sure that changes README.md and metadata.json are merged into master on GitHub. - - stage: ReleaseSnap - displayName: Release Snap - dependsOn: ChangesToMaster - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Publish Snap - jobName: PublishSnapJob - instructions: | - Publish Snap - - stage: ReleaseToMU displayName: Release to MU dependsOn: PushGitTagAndMakeDraftPublic # This only needs the blob to be available @@ -460,7 +438,6 @@ extends: dependsOn: - ReleaseToMU - ReleaseSymbols - - ReleaseSnap jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: diff --git a/.pipelines/PowerShell-vPack-Official.yml b/.pipelines/PowerShell-vPack-Official.yml index 33eddc88d0e..33c72f8963f 100644 --- a/.pipelines/PowerShell-vPack-Official.yml +++ b/.pipelines/PowerShell-vPack-Official.yml @@ -1,5 +1,3 @@ -name: $(BuildDefinitionName)_$(date:yyMM).$(date:dd)$(rev:rrr) - trigger: none parameters: # parameters are shown up in ADO UI in a build queue time @@ -28,6 +26,8 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: 'Release Tag Var:' default: 'fromBranch' +name: vPack_${{ parameters.architecture }}_$(date:yyMM).$(date:dd)$(rev:rrr) + variables: - name: CDP_DEFINITION_BUILD_COUNT value: $[counter('', 0)] @@ -138,22 +138,31 @@ extends: installationPath: $(Agent.ToolsDirectory)/dotnet - pwsh: | - $packageArtifactName = 'drop_windows_package_package_${{ parameters.architecture }}' + $packageArtifactName = 'drop_windows_package_package_win_${{ parameters.architecture }}' $vstsCommandString = "vso[task.setvariable variable=PackageArtifactName]$packageArtifactName" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" - displayName: 'Set package artifact name' + + $packageArtifactPath = '$(Pipeline.Workspace)\PSPackagesOfficial' + $vstsCommandString = "vso[task.setvariable variable=PackageArtifactPath]$packageArtifactPath" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: 'Set package artifact variables' - download: PSPackagesOfficial artifact: $(PackageArtifactName) displayName: Download package - - pwsh: 'Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name' + - pwsh: 'Get-ChildItem $(PackageArtifactPath)\* -recurse | Select-Object -ExpandProperty Name' displayName: 'Capture Artifact Listing' - pwsh: | $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -include *.zip, *.msi | ForEach-Object { + $packages = Get-ChildItem $(PackageArtifactPath)\* -recurse -include *.zip, *.msi + + if($packages.count -eq 0) {throw "No packages found in $(PackageArtifactPath)"} + + $packages | ForEach-Object { if($_.Name -notmatch 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(fxdependent|x64|arm64|x86|fxdependentWinDesktop)\.(msi|zip){1}') { $messageInstance = "$($_.Name) is not a valid package name" @@ -166,7 +175,7 @@ extends: displayName: 'Validate Zip and MSI Package Names' - pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -include *.zip, *.msi | ForEach-Object { + Get-ChildItem $(PackageArtifactPath)\* -recurse -include *.zip | ForEach-Object { if($_.Name -match 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(${{ parameters.architecture }})\.(zip){1}') { Expand-Archive -Path $_.FullName -DestinationPath $(ob_outputDirectory) @@ -197,7 +206,11 @@ extends: - pwsh: | Write-Verbose "VPack Version: $(ob_createvpack_version)" -Verbose - Get-ChildItem -Path $(ob_outputDirectory)\* -Recurse + $vpackFiles = Get-ChildItem -Path $(ob_outputDirectory)\* -Recurse + if($vpackFiles.Count -eq 0) { + throw "No files found in $(ob_outputDirectory)" + } + $vpackFiles displayName: Debug Output Directory and Version condition: succeededOrFailed() @@ -207,5 +220,5 @@ extends: command: 'sign' signing_environment: 'azure-ado' cp_code: $(windows_build_tools_cert_id) - files_to_sign: '**/*.exe;**/*.dll;**/*.ps1;**/*.psm1' + files_to_sign: '**/*.exe;**/System.Management.Automation.dll' search_root: $(ob_outputDirectory) diff --git a/.pipelines/apiscan-gen-notice.yml b/.pipelines/apiscan-gen-notice.yml index f4fd167d7a0..1507b9345bd 100644 --- a/.pipelines/apiscan-gen-notice.yml +++ b/.pipelines/apiscan-gen-notice.yml @@ -13,6 +13,9 @@ parameters: default: false variables: + # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. + # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. + - group: symbols - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: CDP_DEFINITION_BUILD_COUNT @@ -86,8 +89,8 @@ extends: softwareName: "PowerShell" # Default is repo name versionNumber: "7.5" # Default is build number isLargeApp: false # Default: false. + symbolsFolder: $(SymbolsServerUrl);$(ob_outputDirectory) #softwareFolder - relative path to a folder to be scanned. Default value is root of artifacts folder. -#symbolsFolder - relative path to a folder that contains symbols. Default value is root of artifacts folder. tsaOptionsFile: .config\tsaoptions.json diff --git a/.pipelines/templates/checkAzureContainer.yml b/.pipelines/templates/checkAzureContainer.yml index a5ce2b1c666..a6a86214d07 100644 --- a/.pipelines/templates/checkAzureContainer.yml +++ b/.pipelines/templates/checkAzureContainer.yml @@ -51,22 +51,13 @@ jobs: } displayName: 'Check suppress.json' - # Needed as per FAQ here: https://eng.ms/docs/products/onebranch/build/troubleshootingfaqs - - task: PowerShell@2 - displayName: 'Update Az.Storage Module' - inputs: - targetType: 'inline' - script: | - Get-PackageProvider -Name NuGet -ForceBootstrap - Install-Module -Name Az.Storage -Verbose -Force -AllowClobber - Uninstall-AzureRm -Verbose - - task: AzurePowerShell@5 displayName: Check if blob exists and delete if specified inputs: azureSubscription: az-blob-cicd-infra scriptType: inlineScript - azurePowerShellVersion: latestVersion + azurePowerShellVersion: LatestVersion + pwsh: true inline: | $containersToDelete = @('$(AzureVersion)', '$(AzureVersion)-private', '$(AzureVersion)-nuget', '$(AzureVersion)-gc') diff --git a/.pipelines/templates/compliance/apiscan.yml b/.pipelines/templates/compliance/apiscan.yml index b30d72f6a56..17f07a597b5 100644 --- a/.pipelines/templates/compliance/apiscan.yml +++ b/.pipelines/templates/compliance/apiscan.yml @@ -12,9 +12,6 @@ jobs: value: fromBranch # Defines the variables APIScanClient, APIScanTenant and APIScanSecret - group: PS-PS-APIScan - # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. - # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. - - group: symbols - name: branchCounterKey value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - name: branchCounter @@ -61,14 +58,12 @@ jobs: parameters: repoRoot: '$(repoRoot)' - - pwsh: | - Import-Module .\build.psm1 -force - Start-PSBootstrap - workingDirectory: '$(repoRoot)' - retryCountOnTaskFailure: 2 - displayName: 'Bootstrap' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)" - pwsh: | Import-Module .\build.psm1 -force @@ -80,19 +75,6 @@ jobs: workingDirectory: '$(repoRoot)' retryCountOnTaskFailure: 2 - - pwsh: | - $modules = 'Az.Accounts', 'Az.Storage' - foreach($module in $modules) { - if(!(get-module $module -listavailable)) { - Write-Verbose "installing $module..." -verbose - Install-Module $module -force -AllowClobber - } else { - Write-Verbose "$module already installed." -verbose - } - } - displayName: Install PowerShell modules - workingDirectory: '$(repoRoot)' - - task: AzurePowerShell@5 displayName: Download winverify-private Artifacts inputs: @@ -140,10 +122,25 @@ jobs: } Copy-Item -Path "$OutputFolder\*" -Destination '$(ob_outputDirectory)' -Recurse -Verbose - workingDirectory: '$(repoRoot)' displayName: 'Build PowerShell Source' + - pwsh: | + # Only key windows runtimes + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -notmatch '.*\/runtimes\/win'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Force -Verbose -Path $_.FullName + } + + # Temporarily remove runtimes/win-x64 due to issues with that runtime + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -match '.*\/runtimes\/win-x86\/'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Force -Verbose -Path $_.FullName + } + + workingDirectory: '$(repoRoot)' + displayName: 'Remove unused runtimes' + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. displayName: 🔏 CodeQL 3000 Finalize condition: eq(variables['CODEQL_ENABLED'], 'true') diff --git a/.pipelines/templates/compliance/generateNotice.yml b/.pipelines/templates/compliance/generateNotice.yml index 9a00ed6f01d..7de316e8b49 100644 --- a/.pipelines/templates/compliance/generateNotice.yml +++ b/.pipelines/templates/compliance/generateNotice.yml @@ -84,40 +84,6 @@ jobs: displayName: Capture Notice continueOnError: true - - powershell: | - [System.Net.ServicePointManager]::SecurityProtocol = - [System.Net.ServicePointManager]::SecurityProtocol -bor - [System.Security.Authentication.SslProtocols]::Tls12 -bor - [System.Security.Authentication.SslProtocols]::Tls11 - - Set-ItemProperty -Path 'HKLM:\SOFTWARE\Wow6432Node\Microsoft\.NetFramework\v4.0.30319' -Name 'SchUseStrongCrypto' -Value '1' -Type DWord - Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\.NetFramework\v4.0.30319' -Name 'SchUseStrongCrypto' -Value '1' -Type DWord - Get-PackageProvider -Name NuGet -ForceBootstrap - displayName: Initalize PowerShellGet - - - powershell: | - $modules = 'Az.Accounts', 'Az.Storage' - foreach($module in $modules) { - if(!(get-module $module -listavailable)) { - Write-Verbose "installing $module..." -verbose - Install-Module $module -force -AllowClobber - } else { - Write-Verbose "$module already installed." -verbose - #Update-Module $module -verbose - } - } - displayName: Install PowerShell modules - - - powershell: | - if(Get-Command -Name Uninstall-AzureRm -ErrorAction Ignore){ - Write-Verbose "running Uninstall-AzureRm" -verbose - Uninstall-AzureRm - } else { - Write-Verbose "Uninstall-AzureRm not present" -verbose - } - displayName: Uninstall Uninstall-AzureRm - continueOnError: true - - task: AzurePowerShell@5 displayName: Upload Notice inputs: diff --git a/.pipelines/templates/linux-package-build.yml b/.pipelines/templates/linux-package-build.yml index 05c798cd2c3..a9f4833cc1d 100644 --- a/.pipelines/templates/linux-package-build.yml +++ b/.pipelines/templates/linux-package-build.yml @@ -103,7 +103,7 @@ jobs: Import-Module "$repoRoot/build.psm1" Import-Module "$repoRoot/tools/packaging" - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package $psOptionsPath = "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/${unsignedDrop}/psoptions/psoptions.json" diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml index 2e6f9c3e5e3..d6026dc5336 100644 --- a/.pipelines/templates/linux.yml +++ b/.pipelines/templates/linux.yml @@ -62,6 +62,13 @@ jobs: AnalyzeInPipeline: true Language: csharp + - task: UseDotNet@2 + inputs: + useGlobalJson: true + workingDirectory: $(PowerShellRoot) + env: + ob_restore_phase: true + - pwsh: | $runtime = $env:RUNTIME @@ -75,7 +82,6 @@ jobs: Import-Module -Name $(PowerShellRoot)/build.psm1 -Force $buildWithSymbolsPath = New-Item -ItemType Directory -Path $(Pipeline.Workspace)/Symbols_$(Runtime) -Force - Start-PSBootstrap $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose $ReleaseTagParam = @{} diff --git a/.pipelines/templates/mac-package-build.yml b/.pipelines/templates/mac-package-build.yml index 2da8c2b7615..40a6faef06c 100644 --- a/.pipelines/templates/mac-package-build.yml +++ b/.pipelines/templates/mac-package-build.yml @@ -110,7 +110,7 @@ jobs: Write-Verbose -Message "LTS Release: $LTS" } - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package $macosRuntime = "osx-$buildArch" @@ -173,56 +173,43 @@ jobs: Get-ChildItem -Path $(Pipeline.Workspace) -Filter "*.zip" -File | Write-Verbose -Verbose displayName: Compress package files for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'ESRP CodeSigning' + - task: onebranch.pipeline.signing@1 + displayName: 'OneBranch CodeSigning Package' inputs: - ConnectedServiceName: 'ESRPMacOSSigning' - AppRegistrationClientId: '$(AppRegistrationClientId)' - AppRegistrationTenantId: '$(AppRegistrationTenantId)' - AuthAKVName: 'pwsh-CICD-Keyvault' - AuthCertName: 'PS-macos-signing' - AuthSignCertName: 'ESRP-OneCert' # this is not needed for pkg signing - FolderPath: $(Pipeline.Workspace) - Pattern: '*.zip' - signConfigType: inlineSignParams - inlineOperation: | - [{ + command: 'sign' + files_to_sign: '**/*-osx-*.zip' + search_root: '$(Pipeline.Workspace)' + inline_operation: | + [ + { "KeyCode": "$(KeyCode)", - "OperationSetCode": "MacAppDeveloperSign", - "parameters": [ - { - "parameterName": "hardening", - "parameterValue": "enable" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://Microsoft.com" - } - ], + "OperationCode": "MacAppDeveloperSign", "ToolName": "sign", - "ToolVersion": "1.0" - }] - SessionTimeout: 90 - ServiceEndpointUrl: '$(ServiceEndpointUrl)' - MaxConcurrency: 25 + "ToolVersion": "1.0", + "Parameters": { + "Hardening": "Enable", + "OpusInfo": "http://microsoft.com" + } + } + ] - pwsh: | $signedPkg = Get-ChildItem -Path $(Pipeline.Workspace) -Filter "*osx*.zip" -File - + $signedPkg | ForEach-Object { Write-Verbose -Verbose "Signed package zip: $_" - + if (-not (Test-Path $_)) { throw "Package not found: $_" } - - if (-not (Test-Path $env:ob_outputDirectory)) { - $null = New-Item -Path $env:ob_outputDirectory -ItemType Directory + + if (-not (Test-Path $(ob_outputDirectory))) { + $null = New-Item -Path $(ob_outputDirectory) -ItemType Directory } - Expand-Archive -Path $_ -DestinationPath $env:ob_outputDirectory -Verbose + Expand-Archive -Path $_ -DestinationPath $(ob_outputDirectory) -Verbose } Write-Verbose -Verbose "Expanded pkg file:" - Get-ChildItem -Path $env:ob_outputDirectory | Write-Verbose -Verbose + Get-ChildItem -Path $(ob_outputDirectory) | Write-Verbose -Verbose displayName: Expand signed file diff --git a/.pipelines/templates/mac.yml b/.pipelines/templates/mac.yml index 4f9604ea100..310c5695979 100644 --- a/.pipelines/templates/mac.yml +++ b/.pipelines/templates/mac.yml @@ -39,9 +39,16 @@ jobs: sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" displayName: 'Create $(Agent.TempDirectory)/PowerShell' + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(PowerShellRoot) + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package displayName: 'Bootstrap VM' env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) diff --git a/.pipelines/templates/nupkg.yml b/.pipelines/templates/nupkg.yml index be4b704557e..d7837c5c3dc 100644 --- a/.pipelines/templates/nupkg.yml +++ b/.pipelines/templates/nupkg.yml @@ -97,12 +97,17 @@ jobs: - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: '$(PowerShellRoot)' + - pwsh: | Set-Location -Path '$(PowerShellRoot)' Import-Module "$(PowerShellRoot)/build.psm1" -Force - Start-PSBootstrap -Verbose - $sharedModules = @('Microsoft.PowerShell.Commands.Management', 'Microsoft.PowerShell.Commands.Utility', 'Microsoft.PowerShell.ConsoleHost', diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml index 84a02c9e0f0..bfa07c9b27f 100644 --- a/.pipelines/templates/release-MakeBlobPublic.yml +++ b/.pipelines/templates/release-MakeBlobPublic.yml @@ -52,17 +52,6 @@ jobs: Get-ChildItem Env: displayName: 'Capture Environment Variables' - - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } - - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - displayName: Remove AzRM modules - - task: AzurePowerShell@5 displayName: Copy blobs to PSInfra storage inputs: @@ -150,17 +139,6 @@ jobs: Get-ChildItem Env: displayName: 'Capture Environment Variables' - - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } - - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - displayName: Remove AzRM modules - - task: AzurePowerShell@5 displayName: Copy blobs to PSInfra storage inputs: diff --git a/.pipelines/templates/release-SetReleaseTagandContainerName.yml b/.pipelines/templates/release-SetReleaseTagandContainerName.yml index 407a3a8f91d..d40551353d2 100644 --- a/.pipelines/templates/release-SetReleaseTagandContainerName.yml +++ b/.pipelines/templates/release-SetReleaseTagandContainerName.yml @@ -1,3 +1,7 @@ +parameters: +- name: restorePhase + default: false + steps: - pwsh: | $variable = 'releaseTag' @@ -13,6 +17,8 @@ steps: Write-Host -Object "##$vstsCommandString" name: OutputReleaseTag displayName: Set Release Tag + env: + ob_restore_phase: ${{ parameters.restorePhase }} - pwsh: | $azureVersion = '$(OutputReleaseTag.ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' @@ -26,3 +32,5 @@ steps: Write-Host "##$vstsCommandString" name: OutputVersion displayName: Set container name + env: + ob_restore_phase: ${{ parameters.restorePhase }} diff --git a/.pipelines/templates/release-SetTagAndTools.yml b/.pipelines/templates/release-SetTagAndChangelog.yml similarity index 59% rename from .pipelines/templates/release-SetTagAndTools.yml rename to .pipelines/templates/release-SetTagAndChangelog.yml index 7b8a946e323..f0c516dd28f 100644 --- a/.pipelines/templates/release-SetTagAndTools.yml +++ b/.pipelines/templates/release-SetTagAndChangelog.yml @@ -1,6 +1,6 @@ jobs: -- job: SetTagAndTools - displayName: Set Tag and Tools +- job: setTagAndChangelog + displayName: Set Tag and Upload Changelog condition: succeeded() pool: type: windows @@ -19,32 +19,6 @@ jobs: clean: true env: ob_restore_phase: true - - - checkout: PSInternalTools - clean: true - env: - ob_restore_phase: true - - - pwsh: | - New-Item -ItemType Directory -Path '$(Pipeline.Workspace)/ToolArtifact' - Get-ChildItem -Path '$(Build.SourcesDirectory)/Internal-PowerShellTeam-Tools/Scripts' -Filter 'GitHubRelease.psm1' -ErrorAction SilentlyContinue | - Copy-Item -Destination '$(Pipeline.Workspace)/ToolArtifact' -Verbose - displayName: Move GitHub Tool - - - task: onebranch.pipeline.signing@1 - displayName: Sign Tools - inputs: - command: 'sign' - signing_profile: internal_azure_service - files_to_sign: '*.ps1;*.psm1' - search_root: '$(Pipeline.Workspace)/ToolArtifact' - - - pwsh: | - Write-Verbose -Verbose "Creating output directory for release tools: $(ob_outputDirectory)/ToolArtifact" - New-Item -Path $(ob_outputDirectory)/ToolArtifact -ItemType Directory -Force - Get-ChildItem -Path "$(Pipeline.Workspace)/ToolArtifact/*" -Recurse | - Copy-Item -Destination $(ob_outputDirectory)/ToolArtifact -Recurse -Verbose - displayName: Upload Tools - pwsh: | Write-Verbose -Verbose "Release Tag: $(OutputReleaseTag.releaseTag)" diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index 3b1573d9777..751ce1ec5e2 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -27,7 +27,8 @@ jobs: artifact: drop_windows_package_package_win_x86 displayName: Download x86 msix patterns: '**/*.msix' - + + # Finds the makeappx tool on the machine with image: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - pwsh: | $cmd = Get-Command makeappx.exe -ErrorAction Ignore if ($cmd) { @@ -36,18 +37,16 @@ jobs: } else { $toolsDir = '$(Pipeline.Workspace)\releasePipeline\tools' New-Item $toolsDir -Type Directory -Force > $null - Invoke-RestMethod -Uri '$(makeappUrl)' -OutFile "$toolsDir\makeappx.zip" - Expand-Archive "$toolsDir\makeappx.zip" -DestinationPath "$toolsDir\makeappx" -Force - $exePath = "$toolsDir\makeappx\makeappx.exe" - - Write-Verbose -Verbose 'makeappx was installed:' - Get-ChildItem -Path $toolsDir -Recurse + $makeappx = Get-ChildItem -Recurse 'C:\Program Files (x86)\Windows Kits\10\makeappx.exe' | + Where-Object { $_.DirectoryName -match 'x64' } | + Select-Object -Last 1 + $exePath = $makeappx.FullName + Write-Verbose -Verbose 'makeappx was found:' } - $vstsCommandString = "vso[task.setvariable variable=MakeAppxPath]$exePath" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" - displayName: Install makeappx tool + displayName: Find makeappx tool retryCountOnTaskFailure: 1 - pwsh: | @@ -76,17 +75,23 @@ jobs: displayName: Create MsixBundle retryCountOnTaskFailure: 1 + - task: onebranch.pipeline.signing@1 + displayName: Sign MsixBundle + inputs: + command: 'sign' + signing_profile: $(MSIXProfile) + files_to_sign: '**/*.msixbundle' + search_root: '$(BundleDir)' + - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } + $signedBundle = Get-ChildItem -Path $(BundleDir) -Filter "*.msixbundle" -File + Write-Verbose -Verbose "Signed bundle: $signedBundle" - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + Copy-Item -Path $signedBundle -Destination $(ob_outputDirectory) -Verbose - displayName: Remove AzRM modules and install Az.Storage + Write-Verbose -Verbose "Uploaded Bundle:" + Get-ChildItem -Path $(ob_outputDirectory) | Write-Verbose -Verbose + displayName: Upload msixbundle to Artifacts - task: AzurePowerShell@5 displayName: Upload msix to blob diff --git a/.pipelines/templates/release-githubNuget.yml b/.pipelines/templates/release-githubNuget.yml new file mode 100644 index 00000000000..8aca980eff2 --- /dev/null +++ b/.pipelines/templates/release-githubNuget.yml @@ -0,0 +1,171 @@ +parameters: + - name: skipPublish + type: boolean + +jobs: +- job: GithubReleaseDraft + displayName: Create GitHub Release Draft + condition: succeeded() + pool: + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + artifactName: drop_setReleaseTagAndChangelog_SetTagAndChangelog + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages + variables: + - template: ./variable/release-shared.yml@self + parameters: + RELEASETAG: $[ stageDependencies.setReleaseTagAndChangelog.setTagAndChangelog.outputs['OutputReleaseTag.releaseTag'] ] + + steps: + - task: PowerShell@2 + inputs: + targetType: inline + script: | + Write-Verbose -Verbose "Release Tag: $(ReleaseTag)" + Get-ChildItem Env: | Out-String -Stream | Write-Verbose -Verbose + displayName: 'Capture Environment Variables' + + - template: release-install-pwsh.yml + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $Path = "$(Pipeline.Workspace)/GitHubPackages" + $OutputPath = Join-Path $Path 'hashes.sha256' + $packages = Get-ChildItem -Path $Path -Include * -Recurse -File + $checksums = $packages | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent + displayName: Add sha256 hashes + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName + displayName: List all files in the workspace + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + Write-Verbose -Verbose "Available modules: " + Get-Module | Write-Verbose -Verbose + + $filePath = Get-ChildItem -Path "$(Pipeline.Workspace)/CHANGELOG" -Filter '*.md' | Select-Object -First 1 -ExpandProperty FullName + + if (-not (Test-Path $filePath)) { + throw "$filePath not found" + } + + $changelog = Get-Content -Path $filePath + + $headingPattern = "^## \[\d+\.\d+\.\d+" + $headingStartLines = $changelog | Select-String -Pattern $headingPattern | Select-Object -ExpandProperty LineNumber + $startLine = $headingStartLines[0] + $endLine = $headingStartLines[1] - 1 + + $clContent = $changelog | Select-Object -Skip ($startLine-1) -First ($endLine - $startLine) | Out-String + + Write-Verbose -Verbose "Selected content: `n$clContent" + + $releaseNotesFilePath = "$(Pipeline.Workspace)/release-notes.md" + $clContent | Out-File -FilePath $releaseNotesFilePath -Encoding utf8 + + Write-Host "##vso[task.setvariable variable=ReleaseNotesFilePath;]$releaseNotesFilePath" + + #if name has prelease then make prerelease true as a variable + if ($releaseVersion -like '*-*') { + Write-Host "##vso[task.setvariable variable=IsPreRelease;]true" + } else { + Write-Host "##vso[task.setvariable variable=IsPreRelease;]false" + } + displayName: Set variables for GitHub release task + + - pwsh: | + Write-Host "ReleaseNotes content:" + Get-Content "$(Pipeline.Workspace)/release-notes.md" -Raw | Out-String -width 9999 | Write-Host + displayName: Verify Release Notes + + - task: GitHubRelease@1 + inputs: + gitHubConnection: GitHubReleasePAT + repositoryName: PowerShell/PowerShell + target: master + assets: '$(Pipeline.Workspace)/GitHubPackages/*' + tagSource: 'userSpecifiedTag' + tag: '$(ReleaseTag)' + isDraft: true + addChangeLog: false + action: 'create' + releaseNotesFilePath: '$(ReleaseNotesFilePath)' + isPrerelease: '$(IsPreRelease)' + +- job: NuGetPublish + displayName: Publish to NuGet + condition: succeeded() + pool: + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages + variables: + - template: ./variable/release-shared.yml@self + parameters: + VERSION: $[ stageDependencies.setReleaseTagAndChangelog.SetTagAndChangelog.outputs['OutputVersion.Version'] ] + + steps: + - template: release-install-pwsh.yml + + - pwsh: | + Write-Verbose -Verbose "Version: $(Version)" + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + + - pwsh: | + #Exclude all global tool packages. Their names start with 'PowerShell.' + $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" + Copy-Item "$(Pipeline.Workspace)/NuGetPackages/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose + + $releaseVersion = '$(Version)' + $globalToolPath = "$(Pipeline.Workspace)/NuGetPackages/PowerShell.$releaseVersion.nupkg" + + if ($releaseVersion -notlike '*-*') { + # Copy the global tool package for stable releases + Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" + } + + Write-Verbose -Verbose "The .nupkgs below will be pushed:" + Get-ChildItem "$(Pipeline.Workspace)/release" -recurse + displayName: Download and capture nupkgs + condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) + + - task: NuGetCommand@2 + displayName: 'NuGet push' + condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) + inputs: + command: push + packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' + nuGetFeedType: external + publishFeedCredentials: PowerShellNuGetOrgPush diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml deleted file mode 100644 index 31e66b793a4..00000000000 --- a/.pipelines/templates/release-githubtasks.yml +++ /dev/null @@ -1,88 +0,0 @@ -jobs: -- job: GithubReleaseDraft - displayName: Create GitHub Release Draft - condition: succeeded() - pool: - type: release - os: windows - templateContext: - inputs: - - input: pipelineArtifact - artifactName: drop_setReleaseTagAndUploadTools_SetTagAndTools - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_upload_upload_packages - variables: - - template: ./variable/release-shared.yml@self - parameters: - RELEASETAG: $[ stageDependencies.setReleaseTagAndUploadTools.SetTagAndTools.outputs['OutputReleaseTag.releaseTag'] ] - - steps: - - task: PowerShell@2 - inputs: - targetType: inline - script: | - Write-Verbose -Verbose "Release Tag: $(ReleaseTag)" - Get-ChildItem Env: | Out-String -Stream | Write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - - template: release-install-pwsh.yml - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $Path = "$(Pipeline.Workspace)/GitHubPackages" - $OutputPath = Join-Path $Path 'hashes.sha256' - $packages = Get-ChildItem -Path $Path -Include * -Recurse -File - $checksums = $packages | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - $checksums | Out-File -FilePath $OutputPath -Force - $fileContent = Get-Content -Path $OutputPath -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent - displayName: Add sha256 hashes - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName - displayName: List all files in the workspace - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Import-module '$(Pipeline.Workspace)/ToolArtifact/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - Write-Verbose -Verbose "Available modules: " - Get-Module | Write-Verbose -Verbose - - $filePath = Get-ChildItem -Path "$(Pipeline.Workspace)/CHANGELOG" -Filter '*.md' | Select-Object -First 1 -ExpandProperty FullName - - if (-not (Test-Path $filePath)) { - throw "$filePath not found" - } - - $changelog = Get-Content -Path $filePath - - $headingPattern = "^## \[\d+\.\d+\.\d+" - $headingStartLines = $changelog | Select-String -Pattern $headingPattern | Select-Object -ExpandProperty LineNumber - $startLine = $headingStartLines[0] - $endLine = $headingStartLines[1] - 1 - - $clContent = $changelog | Select-Object -Skip ($startLine-1) -First ($endLine - $startLine) | Out-String - - Write-Verbose -Verbose "Selected content: `n$clContent" - - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/GitHubPackages" -Token $(GitHubReleasePat) - displayName: Publish Release Draft diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml new file mode 100644 index 00000000000..cf7982cd5e1 --- /dev/null +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -0,0 +1,237 @@ +parameters: +- name: skipPublish + type: boolean + default: false + +stages: +- stage: PrepForEV2 + displayName: 'Copy and prep all files needed for EV2 stage' + jobs: + - job: CopyEV2FilesToArtifact + displayName: 'Copy EV2 Files to Artifact' + pool: + type: linux + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: repoRoot + value: '$(Build.SourcesDirectory)/PowerShell' + - name: ev2ServiceGroupRootFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' + - name: ev2ParametersFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' + - group: 'mscodehub-code-read-akv' + - group: 'packages.microsoft.com' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + steps: + - checkout: self ## the global setting on lfs didn't work + lfs: false + env: + ob_restore_phase: true + + - template: release-SetReleaseTagandContainerName.yml + parameters: + restorePhase: true + + - pwsh: | + $packageVersion = '$(OutputReleaseTag.ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Set Package version + env: + ob_restore_phase: true + + - pwsh: | + $branch = 'mirror-target' + $gitArgs = "clone", + "--verbose", + "--branch", + "$branch", + "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", + '$(Pipeline.Workspace)/tools' + $gitArgs | Write-Verbose -Verbose + git $gitArgs + displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + env: + ob_restore_phase: true + + - pwsh: | + Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + env: + ob_restore_phase: true + + - pwsh: | + Get-ChildItem '$(Build.SourcesDirectory)' + displayName: 'Capture BuildDirectory' + env: + ob_restore_phase: true + + - pwsh: | + Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Workspace' + env: + ob_restore_phase: true + + - pwsh: | + New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory + displayName: 'Create Parameters folder under EV2Specs folder' + env: + ob_restore_phase: true + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' + displayName: 'Pip Authenticate' + env: + ob_restore_phase: true + + - pwsh: | + python3 -m pip install --upgrade pip + pip --version --verbose + + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $pythonDlFolderPath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath "python_dl" + pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose + displayName: 'Download pmc-cli package' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_deb' + displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_rpm' + displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_x64' + displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_arm64' + displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - pwsh: | + Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" + $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' + $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' + New-Item -Path $srcFilesFolder -ItemType Directory + $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' + New-Item -Path $packagesFolder -ItemType Directory + + $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm + foreach ($file in $packageFiles) + { + Write-Verbose -Verbose "copying file: $($file.FullName)" + Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose + } + + $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' + tar -czvf $packagesTarGzDestination -C $packagesFolder . + displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' + env: + ob_restore_phase: true + + - pwsh: | + $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' + + $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json + $metadataHash = @{} + $skipPublishValue = '${{ parameters.skipPublish }}' + $metadataHash["ReleaseTag"] = '$(OutputReleaseTag.ReleaseTag)' + $metadataHash["LTS"] = $metadata.LTSRelease.Latest + $metadataHash["ForProduction"] = $true + $metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue) + + $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile + + $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' + $mappingFilePathExists = Test-Path $mappingFilePath + $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" + Write-Verbose -Verbose "Copy mapping.json file at: $mappingFilePath which exists: $mappingFilePathExists to: $mappingFileEV2Path" + Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path + displayName: 'Create pmcScriptMetadata.json and mapping.json file' + env: + ob_restore_phase: true + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)' + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile + displayName: 'Replace values in RolloutSpecPath.json' + env: + ob_restore_phase: true + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + + $identityString = "/subscriptions/$(PmcSubscription)/resourcegroups/$(PmcResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(PmcMIName)" + $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile + displayName: 'Replace values in UploadLinux.Rollout.json file' + env: + ob_restore_phase: true + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(PmcResourceGroup)' + $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(PmcSubscription)' + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile + displayName: 'Replace values in ServiceModel.json' + env: + ob_restore_phase: true + + - pwsh: | + $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' + New-Item -Path $settingFilePath -ItemType File + $pmcMIClientID = '$(PmcMIClientID)' + $pmcEndpoint = '$(PmcEndpointUrl)' + + Add-Content -Path $settingFilePath -Value "[default]" + Add-Content -Path $settingFilePath -Value "base_url = `"$pmcEndpoint`"" + Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" + Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" + displayName: 'Create settings.toml file with MI clientId populated' + env: + ob_restore_phase: true + + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + displayName: Sign Run.ps1 + + - pwsh: | + # folder to tar must have: Run.ps1, settings.toml, python_dl + $srcPath = Join-Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path $srcPath -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + displayName: 'Create archive for the shell extension' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(repoRoot)/.pipelines' + Contents: 'EV2Specs/**' + TargetFolder: $(ob_outputDirectory) diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml deleted file mode 100644 index 98249844d4c..00000000000 --- a/.pipelines/templates/release-publish-nuget.yml +++ /dev/null @@ -1,58 +0,0 @@ -parameters: - - name: skipPublish - default: false - type: boolean - -jobs: -- job: NuGetPublish - displayName: Publish to NuGet - condition: succeeded() - pool: - type: release - os: windows - templateContext: - inputs: - - input: pipelineArtifact - artifactName: drop_setReleaseTagAndUploadTools_SetTagAndTools - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_upload_upload_packages - variables: - - template: ./variable/release-shared.yml@self - parameters: - VERSION: $[ stageDependencies.setReleaseTagAndUploadTools.SetTagAndTools.outputs['OutputVersion.Version'] ] - - steps: - - template: release-install-pwsh.yml - - - pwsh: | - Write-Verbose -Verbose "Version: $(Version)" - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - - pwsh: | - #Exclude all global tool packages. Their names start with 'PowerShell.' - $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" - Copy-Item "$(Pipeline.Workspace)/NuGetPackages/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose - - $releaseVersion = '$(Version)' - $globalToolPath = "$(Pipeline.Workspace)/NuGetPackages/PowerShell.$releaseVersion.nupkg" - - if ($releaseVersion -notlike '*-*') { - # Copy the global tool package for stable releases - Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" - } - - Write-Verbose -Verbose "The .nupkgs below will be pushed:" - Get-ChildItem "$(Pipeline.Workspace)/release" -recurse - displayName: Download and capture nupkgs - condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - - - task: NuGetCommand@2 - displayName: 'NuGet push' - condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - inputs: - command: push - packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' - nuGetFeedType: external - publishFeedCredentials: PowerShellNuGetOrgPush diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 27311611e61..d5454845211 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -1,90 +1,37 @@ -parameters: - - name: skipPublish - default: false - type: boolean - -jobs: -- job: PMCPublish - displayName: Publish to PMC - condition: succeeded() - pool: - type: linux - isCustom: true - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMSUbuntu20.04-Secure +stages: +- stage: 'Prod_Release' + displayName: 'Deploy packages to PMC with EV2' + dependsOn: + - PrepForEV2 variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: 'mscodehub-code-read-akv' - - group: 'packages.microsoft.com' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - steps: - - checkout: self ## the global setting on lfs didn't work - lfs: false - - - template: release-SetReleaseTagAndContainerName.yml - - - pwsh: | - $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Set Package version - - - pwsh: | - $branch = 'mirror-target' - $gitArgs = "clone", - "--verbose", - "--branch", - "$branch", - "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", - '$(Pipeline.Workspace)/tools' - $gitArgs | Write-Verbose -Verbose - git $gitArgs - displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'pmc' - pythonDownloadServiceConnections: pmcDownload - - - pwsh: | - pip install pmc-cli==1.12.0 - - $newPath = (resolve-path '~/.local/bin').providerpath - $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Install pmc cli - - - pwsh: | - $metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json - $params = @{ - ReleaseTag = "$(ReleaseTag)" - AadClientId = "$(PmcCliClientID)" - BlobFolderName = "$(ReleaseTag)" - LTS = $metadata.LTSRelease.Latest - ForProduction = $true - SkipPublish = $${{ parameters.skipPublish }} - MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' - } - - $params | Out-String -width 9999 -Stream | write-Verbose -Verbose - - & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params - displayName: Run release script + - name: ob_release_environment + value: "Production" + - name: repoRoot + value: $(Build.SourcesDirectory) + jobs: + - job: Prod_ReleaseJob + displayName: Publish to PMC + pool: + type: release + + steps: + - task: DownloadPipelineArtifact@2 + inputs: + targetPath: '$(Pipeline.Workspace)' + artifact: drop_PrepForEV2_CopyEv2FilesToArtifact + displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)' + displayName: 'Download to get EV2 Files' + + - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 + displayName: 'Ev2: Push to PMC' + inputs: + UseServerMonitorTask: true + EndpointProviderType: ApprovalService + ApprovalServiceEnvironment: Production + ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' + RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' diff --git a/.pipelines/templates/release-upload-buildinfo.yml b/.pipelines/templates/release-upload-buildinfo.yml index ea7b90db8e3..d35630168a0 100644 --- a/.pipelines/templates/release-upload-buildinfo.yml +++ b/.pipelines/templates/release-upload-buildinfo.yml @@ -104,17 +104,6 @@ jobs: } displayName: Create json files - - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } - - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - displayName: Remove AzRM modules - - task: AzurePowerShell@5 displayName: Upload buildjson to blob inputs: diff --git a/.pipelines/templates/release-validate-fxdpackages.yml b/.pipelines/templates/release-validate-fxdpackages.yml index 62e907fcf36..53657e6414a 100644 --- a/.pipelines/templates/release-validate-fxdpackages.yml +++ b/.pipelines/templates/release-validate-fxdpackages.yml @@ -1,10 +1,25 @@ parameters: - jobName: "" - displayName: "" - jobtype: "" - artifactName: "" - packageNamePattern: "" - arm64: "no" + - name: jobName + type: string + default: "" + - name: displayName + type: string + default: "" + - name: jobtype + type: string + default: "" + - name: artifactName + type: string + default: "" + - name: packageNamePattern + type: string + default: "" + - name: arm64 + type: string + default: "no" + - name: enableCredScan + type: boolean + default: true jobs: - job: ${{ parameters.jobName }} @@ -19,6 +34,8 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - name: ob_sdl_tsa_configFile value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_enabled + value: ${{ parameters.enableCredScan }} pool: type: ${{ parameters.jobtype }} @@ -44,38 +61,12 @@ jobs: Get-ChildItem "$(Pipeline.Workspace)/PSPackagesOfficial/$artifactName" -Recurse displayName: 'Capture Downloaded Artifacts' - - pwsh: | - $repoRoot = "$(Build.SourcesDirectory)/PowerShell" - $dotnetMetadataPath = "$repoRoot/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$repoRoot/global.json" -Raw | ConvertFrom-Json).sdk.version - Import-Module "$repoRoot/build.psm1" -Force - - Find-Dotnet - - if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) - { - $nugetFeed = ([xml](Get-Content $repoRoot/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value - if ($nugetFeed) { - Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet - Write-Verbose -Message "Register new package source 'dotnet'" -verbose - } - } - - ## Install latest version from the channel - - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion - Start-PSBootstrap - - Write-Verbose -Message "Installing .NET SDK completed." -Verbose - - displayName: Install .NET - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)/PowerShell" - pwsh: | $artifactName = '$(artifactName)' diff --git a/.pipelines/templates/release-validate-globaltools.yml b/.pipelines/templates/release-validate-globaltools.yml index 0820e5591f6..3c88a278791 100644 --- a/.pipelines/templates/release-validate-globaltools.yml +++ b/.pipelines/templates/release-validate-globaltools.yml @@ -38,44 +38,15 @@ jobs: Get-ChildItem "$(Pipeline.Workspace)/PSPackagesOfficial/drop_nupkg_build_nupkg" -Recurse displayName: 'Capture Downloaded Artifacts' - - pwsh: | - $repoRoot = "$(Build.SourcesDirectory)/PowerShell" - $dotnetMetadataPath = "$repoRoot/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$repoRoot/global.json" -Raw | ConvertFrom-Json).sdk.version - Import-Module "$repoRoot/build.psm1" -Force - - Find-Dotnet - - if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) - { - $nugetFeed = ([xml](Get-Content $repoRoot/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value - if ($nugetFeed) { - Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet - Write-Verbose -Message "Register new package source 'dotnet'" -verbose - } - } - - ## Install latest version from the channel - - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion - Start-PSBootstrap - - Write-Verbose -Message "Installing .NET SDK completed." -Verbose - - displayName: Install .NET - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(REPOROOT) - pwsh: | $repoRoot = "$(Build.SourcesDirectory)/PowerShell" - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - Import-Module "$repoRoot/build.psm1" -Force - Start-PSBootstrap $toolPath = New-Item -ItemType Directory "$(System.DefaultWorkingDirectory)/toolPath" | Select-Object -ExpandProperty FullName @@ -108,8 +79,6 @@ jobs: - pwsh: | $repoRoot = "$(Build.SourcesDirectory)/PowerShell" - Import-Module "$repoRoot/build.psm1" -Force - Start-PSBootstrap $exeName = if ($IsWindows) { "pwsh.exe" } else { "pwsh" } diff --git a/.pipelines/templates/release-validate-packagenames.yml b/.pipelines/templates/release-validate-packagenames.yml index 3e2987591aa..1eaf9c070ee 100644 --- a/.pipelines/templates/release-validate-packagenames.yml +++ b/.pipelines/templates/release-validate-packagenames.yml @@ -28,18 +28,6 @@ jobs: Write-Host "##vso[build.updatebuildnumber]$name" displayName: Set Release Name - - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } - - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - - displayName: Remove AzRM modules and install Az.Storage - - task: AzurePowerShell@5 displayName: Upload packages to blob inputs: diff --git a/.pipelines/templates/release-validate-sdk.yml b/.pipelines/templates/release-validate-sdk.yml index 3f365f5ebb9..d879ab7f06e 100644 --- a/.pipelines/templates/release-validate-sdk.yml +++ b/.pipelines/templates/release-validate-sdk.yml @@ -46,47 +46,17 @@ jobs: Get-ChildItem "$(Pipeline.Workspace)/PSPackagesOfficial/drop_nupkg_build_nupkg" -Recurse displayName: 'Capture Downloaded Artifacts' - - pwsh: | - $repoRoot = "$(Build.SourcesDirectory)" - - $dotnetMetadataPath = "$repoRoot/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$repoRoot/global.json" -Raw | ConvertFrom-Json).sdk.version - Import-Module "$repoRoot/build.psm1" -Force - - Find-Dotnet - - if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) - { - $nugetFeed = ([xml](Get-Content $repoRoot/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value - - if ($nugetFeed) { - Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet - Write-Verbose -Message "Register new package source 'dotnet'" -verbose - } - } - - ## Install latest version from the channel - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion - - Start-PSBootstrap - - Write-Verbose -Message "Installing .NET SDK completed." -Verbose - - displayName: Install .NET - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(REPOROOT) - pwsh: | $repoRoot = "$(Build.SourcesDirectory)" $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - Import-Module "$repoRoot/build.psm1" -Force - Start-PSBootstrap $localLocation = "$(Pipeline.Workspace)/PSPackagesOfficial/drop_nupkg_build_nupkg" $xmlElement = @" @@ -116,10 +86,6 @@ jobs: Get-Content $nugetPath - # Add workaround to unblock xUnit testing see issue: https://github.com/dotnet/sdk/issues/26462 - $dotnetPath = if ($IsWindows) { "$env:LocalAppData\Microsoft\dotnet" } else { "$env:HOME/.dotnet" } - $env:DOTNET_ROOT = $dotnetPath - dotnet --info dotnet restore dotnet test /property:RELEASE_VERSION=$releaseVersion --test-adapter-path:. "--logger:xunit;LogFilePath=$(System.DefaultWorkingDirectory)/test-hosting.xml" diff --git a/.pipelines/templates/testartifacts.yml b/.pipelines/templates/testartifacts.yml index 039e9336d7c..240ceae80f7 100644 --- a/.pipelines/templates/testartifacts.yml +++ b/.pipelines/templates/testartifacts.yml @@ -30,12 +30,13 @@ jobs: repoRoot: $(Build.SourcesDirectory)/PowerShell ob_restore_phase: true - - pwsh: | - Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 - Start-PSBootstrap - displayName: Bootstrap + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)/PowerShell" env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) ob_restore_phase: true - pwsh: | @@ -97,12 +98,13 @@ jobs: repoRoot: $(Build.SourcesDirectory)/PowerShell ob_restore_phase: true - - pwsh: | - Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 - Start-PSBootstrap - displayName: Bootstrap + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)/PowerShell" env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) ob_restore_phase: true - pwsh: | diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 35a11ec383c..0994c7ef2b0 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -6,7 +6,7 @@ jobs: type: windows variables: - name: ob_sdl_sbom_enabled - value: false + value: true - name: runCodesignValidationInjection value: false - name: NugetSecurityAnalysisWarningLevel @@ -241,7 +241,7 @@ jobs: Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + Where-Object { $_.Extension -notin '.msix', '.nupkg' -and $_.Name -notmatch '-gc'} | Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse -Verbose Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" @@ -256,18 +256,6 @@ jobs: New-Item -Path $(Build.ArtifactStagingDirectory)/uploaded -ItemType Directory -Force displayName: Create output directory for packages - - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } - - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - - displayName: Remove AzRM modules - - task: AzurePowerShell@5 displayName: Upload packages to blob inputs: diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 735ed7cc48f..8f8273f4ec8 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -63,6 +63,13 @@ jobs: AnalyzeInPipeline: true Language: csharp + - task: UseDotNet@2 + inputs: + useGlobalJson: true + workingDirectory: $(PowerShellRoot) + env: + ob_restore_phase: true + - pwsh: | $runtime = switch ($env:Architecture) { @@ -86,7 +93,7 @@ jobs: Import-Module -Name $(PowerShellRoot)/build.psm1 -Force $buildWithSymbolsPath = New-Item -ItemType Directory -Path $(Pipeline.Workspace)/Symbols_$(Architecture) -Force - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose $ReleaseTagParam = @{} @@ -135,7 +142,6 @@ jobs: } Import-Module -Name $(PowerShellRoot)/build.psm1 -Force - Start-PSBootstrap ## Build global tool Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose @@ -229,8 +235,6 @@ jobs: After that, we repack using Compress-Archive and rename it back to a nupkg. #> - Import-Module -Name $(PowerShellRoot)/build.psm1 -Force - Start-PSBootstrap $packagingStrings = Import-PowerShellDataFile "$(PowerShellRoot)\tools\packaging\packaging.strings.psd1" $outputPath = Join-Path '$(ob_outputDirectory)' 'globaltool' diff --git a/.pipelines/templates/windows-package-build.yml b/.pipelines/templates/windows-package-build.yml index da2579d75a3..08dd15fc79f 100644 --- a/.pipelines/templates/windows-package-build.yml +++ b/.pipelines/templates/windows-package-build.yml @@ -78,6 +78,13 @@ jobs: env: ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + - task: UseDotNet@2 + inputs: + useGlobalJson: true + workingDirectory: $(REPOROOT) + env: + ob_restore_phase: true + - pwsh: | $msixUrl = '$(makeappUrl)' Invoke-RestMethod -Uri $msixUrl -OutFile '$(Pipeline.Workspace)\makeappx.zip' @@ -105,7 +112,7 @@ jobs: Import-Module "$repoRoot\build.psm1" Import-Module "$repoRoot\tools\packaging" - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package $signedFilesPath, $psoptionsFilePath = if ($env:RUNTIME -eq 'minsize') { "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_x64_${runtime}\$signedFolder" @@ -136,7 +143,7 @@ jobs: Write-Verbose -Message "LTS Release: $LTS" } - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package $WindowsRuntime = switch ($runtime) { 'x64' { 'win7-x64' } diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000000..222861c3415 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 2, + "useTabs": false +} diff --git a/.vsts-ci/linux-internal.yml b/.vsts-ci/linux-internal.yml new file mode 100644 index 00000000000..6286a03fb52 --- /dev/null +++ b/.vsts-ci/linux-internal.yml @@ -0,0 +1,116 @@ +# Pipeline to run Linux CI internally +name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + # Batch merge builds together while a merge build is running + batch: true + branches: + include: + - master + - release* + - feature* + paths: + include: + - '*' + exclude: + - .vsts-ci/misc-analysis.yml + - .github/ISSUE_TEMPLATE/* + - .github/workflows/* + - .dependabot/config.yml + - .pipelines/* + - test/perf/* +pr: + branches: + include: + - master + - release* + - feature* + paths: + include: + - '*' + exclude: + - .dependabot/config.yml + - .github/ISSUE_TEMPLATE/* + - .github/workflows/* + - .vsts-ci/misc-analysis.yml + - .vsts-ci/windows.yml + - .vsts-ci/windows/* + - tools/cgmanifest.json + - LICENSE.txt + - test/common/markdown/* + - test/perf/* + - tools/releaseBuild/* + - tools/install* + - tools/releaseBuild/azureDevOps/templates/* + - README.md + - .spelling + - .pipelines/* + +variables: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + # Avoid expensive initialization of dotnet cli, see: https://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + __SuppressAnsiEscapeSequences: 1 + nugetMultiFeedWarnLevel: none + +resources: + repositories: + - repository: Docker + type: github + endpoint: PowerShell + name: PowerShell/PowerShell-Docker + ref: master + +stages: +- stage: BuildLinuxStage + displayName: Build for Linux + jobs: + - template: templates/ci-build.yml + parameters: + pool: ubuntu-20.04 + jobName: linux_build + displayName: linux Build + +- stage: TestUbuntu + displayName: Test for Ubuntu + dependsOn: [BuildLinuxStage] + jobs: + - template: templates/nix-test.yml + parameters: + name: Ubuntu + pool: ubuntu-20.04 + purpose: UnelevatedPesterTests + tagSet: CI + + - template: templates/nix-test.yml + parameters: + name: Ubuntu + pool: ubuntu-20.04 + purpose: ElevatedPesterTests + tagSet: CI + + - template: templates/nix-test.yml + parameters: + name: Ubuntu + pool: ubuntu-20.04 + purpose: UnelevatedPesterTests + tagSet: Others + + - template: templates/nix-test.yml + parameters: + name: Ubuntu + pool: ubuntu-20.04 + purpose: ElevatedPesterTests + tagSet: Others + + - template: templates/verify-xunit.yml + parameters: + pool: ubuntu-20.04 + +- stage: PackageLinux + displayName: Package Linux + dependsOn: ["BuildLinuxStage"] + jobs: + - template: linux/templates/packaging.yml + parameters: + pool: ubuntu-20.04 diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index c1a1fd5c0ab..338821e37dd 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -34,24 +34,16 @@ pr: - feature* paths: include: - - '*' - exclude: - - .dependabot/config.yml - - .github/ISSUE_TEMPLATE/* - - .github/workflows/* - - .vsts-ci/misc-analysis.yml - - .vsts-ci/windows.yml - - .vsts-ci/windows/* - - tools/cgmanifest.json - - LICENSE.txt - - test/common/markdown/* - - test/perf/* - - tools/releaseBuild/* - - tools/install* - - tools/releaseBuild/azureDevOps/templates/* - - README.md - - .spelling - - .pipelines/* + - .vsts-ci/linux.yml + - .vsts-ci/linux/templates/packaging.yml + - assets/manpage/* + - build.psm1 + - global.json + - nuget.config + - PowerShell.Common.props + - src/*.csproj + - tools/ci.psm1 + - tools/packaging/* variables: DOTNET_CLI_TELEMETRY_OPTOUT: 1 @@ -79,93 +71,6 @@ stages: jobName: linux_build displayName: linux Build -- stage: TestUbuntu - displayName: Test for Ubuntu - dependsOn: [BuildLinuxStage] - jobs: - - template: templates/nix-test.yml - parameters: - name: Ubuntu - pool: ubuntu-20.04 - purpose: UnelevatedPesterTests - tagSet: CI - - - template: templates/nix-test.yml - parameters: - name: Ubuntu - pool: ubuntu-20.04 - purpose: ElevatedPesterTests - tagSet: CI - - - template: templates/nix-test.yml - parameters: - name: Ubuntu - pool: ubuntu-20.04 - purpose: UnelevatedPesterTests - tagSet: Others - - - template: templates/nix-test.yml - parameters: - name: Ubuntu - pool: ubuntu-20.04 - purpose: ElevatedPesterTests - tagSet: Others - - - template: templates/verify-xunit.yml - parameters: - pool: ubuntu-20.04 - -- stage: TestContainer - displayName: Test in a container - dependsOn: [BuildLinuxStage] - jobs: - - job: getContainerJob - displayName: Choose a container - pool: - vmImage: ubuntu-20.04 - steps: - - checkout: self - clean: true - - - checkout: Docker - clean: true - - - pwsh: | - # Initialize container test stage - Import-Module ./PowerShell/tools/ci.psm1 - Invoke-InitializeContainerStage -ContainerPattern '${{ parameters.ContainerPattern }}' - name: getContainerTask - displayName: Initialize Container Stage - continueOnError: true - - - template: templates/test/nix-container-test.yml - parameters: - name: container - pool: ubuntu-20.04 - purpose: UnelevatedPesterTests - tagSet: CI - - - template: templates/test/nix-container-test.yml - parameters: - name: container - pool: ubuntu-20.04 - purpose: ElevatedPesterTests - tagSet: CI - - - template: templates/test/nix-container-test.yml - parameters: - name: container - pool: ubuntu-20.04 - purpose: UnelevatedPesterTests - tagSet: Others - - - template: templates/test/nix-container-test.yml - parameters: - name: container - pool: ubuntu-20.04 - purpose: ElevatedPesterTests - tagSet: Others - - stage: PackageLinux displayName: Package Linux dependsOn: ["BuildLinuxStage"] diff --git a/.vsts-ci/linux/templates/packaging.yml b/.vsts-ci/linux/templates/packaging.yml index fab2e1101fa..e6294951cb6 100644 --- a/.vsts-ci/linux/templates/packaging.yml +++ b/.vsts-ci/linux/templates/packaging.yml @@ -13,6 +13,12 @@ jobs: displayName: ${{ parameters.name }} packaging steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + - pwsh: | Get-ChildItem -Path env: displayName: Capture Environment @@ -33,7 +39,7 @@ jobs: - pwsh: | Import-Module .\build.psm1 - Start-PSBootstrap -Package + Start-PSBootstrap -Scenario Package displayName: Bootstrap - pwsh: | diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index bfb0b3afd21..05d6d71ea71 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -110,6 +110,6 @@ stages: clean: true - pwsh: | import-module ./build.psm1 - start-psbootstrap -package + start-psbootstrap -Scenario package displayName: Bootstrap packaging condition: succeededOrFailed() diff --git a/.vsts-ci/psresourceget-acr.yml b/.vsts-ci/psresourceget-acr.yml index c4211d35d95..1a24983b5b5 100644 --- a/.vsts-ci/psresourceget-acr.yml +++ b/.vsts-ci/psresourceget-acr.yml @@ -137,7 +137,6 @@ stages: - pwsh: | Import-Module .\build.psm1 -force - Start-PSBootstrap Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' $options = (Get-PSOptions) diff --git a/.vsts-ci/templates/ci-build.yml b/.vsts-ci/templates/ci-build.yml index 59d63002567..2c2fbe8d91d 100644 --- a/.vsts-ci/templates/ci-build.yml +++ b/.vsts-ci/templates/ci-build.yml @@ -57,6 +57,12 @@ jobs: - ${{ if ne(variables['UseAzDevOpsFeed'], '') }}: - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + - pwsh: | Import-Module .\tools\ci.psm1 Invoke-CIInstall -SkipUser diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index ab3985dacd6..214ae14b2c6 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -13,6 +13,12 @@ jobs: displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + - template: ./test/nix-test-steps.yml parameters: purpose: ${{ parameters.purpose }} diff --git a/.vsts-ci/templates/test/nix-container-test.yml b/.vsts-ci/templates/test/nix-container-test.yml index 931af6fc675..37c60a4c53b 100644 --- a/.vsts-ci/templates/test/nix-container-test.yml +++ b/.vsts-ci/templates/test/nix-container-test.yml @@ -23,6 +23,12 @@ jobs: displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + - template: ./nix-test-steps.yml parameters: purpose: ${{ parameters.purpose }} diff --git a/.vsts-ci/templates/windows-test.yml b/.vsts-ci/templates/windows-test.yml index 50ff67a32a8..02a8ddd1ea8 100644 --- a/.vsts-ci/templates/windows-test.yml +++ b/.vsts-ci/templates/windows-test.yml @@ -54,6 +54,13 @@ jobs: displayName: 'Capture Artifacts Directory' continueOnError: true + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)" + # must be run frow Windows PowerShell - powershell: | # Remove "Program Files\dotnet" from the env variable PATH, so old SDKs won't affect us. @@ -74,7 +81,6 @@ jobs: - pwsh: | Import-Module .\build.psm1 -force - Start-PSBootstrap Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' $options = (Get-PSOptions) diff --git a/.vsts-ci/windows-daily.yml b/.vsts-ci/windows-daily.yml index 4abcf8ec966..5a2f5ed2425 100644 --- a/.vsts-ci/windows-daily.yml +++ b/.vsts-ci/windows-daily.yml @@ -93,6 +93,13 @@ stages: displayName: Bootstrap condition: succeededOrFailed() + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)" + - pwsh: | Import-Module .\build.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' @@ -104,7 +111,6 @@ stages: - pwsh: | Import-Module .\build.psm1 - Start-PSBootstrap Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose UnelevatedPesterTests -TagSet CI @@ -113,7 +119,6 @@ stages: - pwsh: | Import-Module .\build.psm1 - Start-PSBootstrap Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose ElevatedPesterTests -TagSet CI @@ -122,7 +127,6 @@ stages: - pwsh: | Import-Module .\build.psm1 - Start-PSBootstrap Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose UnelevatedPesterTests -TagSet Others @@ -131,7 +135,6 @@ stages: - pwsh: | Import-Module .\build.psm1 - Start-PSBootstrap Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose ElevatedPesterTests -TagSet Others diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index ca5352cb4e2..c0f08f54a41 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -25,22 +25,18 @@ pr: - feature* paths: include: - - '*' + - .vsts-ci/templates/* + - .vsts-ci/windows.yml + - '*.props' + - build.psm1 + - src/* + - test/* + - tools/buildCommon/* + - tools/ci.psm1 + - tools/WindowsCI.psm1 exclude: - - .dependabot/config.yml - - .github/ISSUE_TEMPLATE/* - - .github/workflows/* - - .vsts-ci/misc-analysis.yml - - tools/cgmanifest.json - - LICENSE.txt - test/common/markdown/* - test/perf/* - - tools/packaging/* - - tools/releaseBuild/* - - tools/releaseBuild/azureDevOps/templates/* - - README.md - - .spelling - - .pipelines/* variables: GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'" diff --git a/.vsts-ci/windows/templates/windows-packaging.yml b/.vsts-ci/windows/templates/windows-packaging.yml index 84b02d14dfd..cc32837a304 100644 --- a/.vsts-ci/windows/templates/windows-packaging.yml +++ b/.vsts-ci/windows/templates/windows-packaging.yml @@ -47,9 +47,6 @@ jobs: displayName: Capture PowerShell Version Table condition: succeededOrFailed() - - - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - - pwsh: | Import-Module .\tools\ci.psm1 Switch-PSNugetConfig -Source Public @@ -57,6 +54,13 @@ jobs: condition: succeeded() workingDirectory: $(repoPath) + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(repoPath) + - pwsh: | Import-Module .\tools\ci.psm1 Invoke-CIInstall -SkipUser diff --git a/CHANGELOG/7.5.md b/CHANGELOG/7.5.md index 702c60346e7..37aa3e27995 100644 --- a/CHANGELOG/7.5.md +++ b/CHANGELOG/7.5.md @@ -1,5 +1,84 @@ # 7.5 Changelog +## [7.5.1] + +### Engine Updates and Fixes + +- Fallback to AppLocker after `WldpCanExecuteFile` (#25305) + +### Code Cleanup + +
+ + + +
+ +### Tools + +- Do not run labels workflow in the internal repository (#25343) +- Update `CODEOWNERS` (#25321) +- Check GitHub token availability for `Get-Changelog` (#25328) +- Update PowerShell team members in `releaseTools.psm1` (#25302) + +### Build and Packaging Improvements + +
+ + + +

Update to .NET SDK 9.0.203

+ + + + + +
+ +[7.5.1]: https://github.com/PowerShell/PowerShell/compare/v7.5.0...v7.5.1 + ## [7.5.0] ### Build and Packaging Improvements diff --git a/DotnetRuntimeMetadata.json b/DotnetRuntimeMetadata.json index 644b79dcd7a..f78f43e709b 100644 --- a/DotnetRuntimeMetadata.json +++ b/DotnetRuntimeMetadata.json @@ -4,7 +4,7 @@ "quality": "daily", "qualityFallback": "preview", "packageVersionPattern": "9.0.0-preview.6", - "sdkImageVersion": "9.0.102", + "sdkImageVersion": "9.0.203", "nextChannel": "9.0.0-preview.7", "azureFeed": "", "sdkImageOverride": "" diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt index 4abb1717d67..d48be300df3 100644 --- a/ThirdPartyNotices.txt +++ b/ThirdPartyNotices.txt @@ -284,7 +284,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI --------------------------------------------------------- -Microsoft.Extensions.ObjectPool 8.0.12 - MIT +Microsoft.Extensions.ObjectPool 8.0.15 - MIT Copyright Jorn Zaefferer @@ -354,7 +354,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI --------------------------------------------------------- -Microsoft.Security.Extensions 1.3.0 - MIT +Microsoft.Security.Extensions 1.4.0 - MIT (c) Microsoft Corporation @@ -446,7 +446,7 @@ SOFTWARE. --------------------------------------------------------- -Microsoft.Win32.Registry.AccessControl 9.0.1 - MIT +Microsoft.Win32.Registry.AccessControl 9.0.4 - MIT Copyright (c) 2021 @@ -536,7 +536,7 @@ SOFTWARE. --------------------------------------------------------- -Microsoft.Win32.SystemEvents 9.0.1 - MIT +Microsoft.Win32.SystemEvents 9.0.4 - MIT Copyright (c) 2021 @@ -626,7 +626,7 @@ SOFTWARE. --------------------------------------------------------- -Microsoft.Windows.Compatibility 9.0.1 - MIT +Microsoft.Windows.Compatibility 9.0.4 - MIT (c) Microsoft Corporation @@ -679,7 +679,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------- -runtime.android-arm.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.android-arm.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -769,7 +769,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.android-arm64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.android-arm64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -859,7 +859,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.android-x64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.android-x64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -949,7 +949,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.android-x86.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.android-x86.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1039,7 +1039,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-arm.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-arm.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1129,7 +1129,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-arm64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-arm64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1219,7 +1219,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-bionic-arm64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-bionic-arm64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1309,7 +1309,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-bionic-x64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-bionic-x64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1399,7 +1399,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-musl-arm.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-musl-arm.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1489,7 +1489,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-musl-arm64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-musl-arm64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1579,7 +1579,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-musl-x64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-musl-x64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1669,7 +1669,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.linux-x64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.linux-x64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1759,7 +1759,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.maccatalyst-arm64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.maccatalyst-arm64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1849,7 +1849,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.maccatalyst-x64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.maccatalyst-x64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -1992,7 +1992,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -2082,7 +2082,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.osx-arm64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.osx-arm64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -2172,7 +2172,7 @@ SOFTWARE. --------------------------------------------------------- -runtime.osx-x64.runtime.native.System.IO.Ports 9.0.1 - MIT +runtime.osx-x64.runtime.native.System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -2262,7 +2262,7 @@ SOFTWARE. --------------------------------------------------------- -System.CodeDom 9.0.1 - MIT +System.CodeDom 9.0.4 - MIT Copyright (c) 2021 @@ -2438,7 +2438,7 @@ SOFTWARE. --------------------------------------------------------- -System.ComponentModel.Composition 9.0.1 - MIT +System.ComponentModel.Composition 9.0.4 - MIT Copyright (c) 2021 @@ -2528,7 +2528,7 @@ SOFTWARE. --------------------------------------------------------- -System.ComponentModel.Composition.Registration 9.0.1 - MIT +System.ComponentModel.Composition.Registration 9.0.4 - MIT Copyright (c) 2021 @@ -2618,7 +2618,7 @@ SOFTWARE. --------------------------------------------------------- -System.Configuration.ConfigurationManager 9.0.1 - MIT +System.Configuration.ConfigurationManager 9.0.4 - MIT Copyright (c) 2021 @@ -2708,7 +2708,7 @@ SOFTWARE. --------------------------------------------------------- -System.Data.Odbc 9.0.1 - MIT +System.Data.Odbc 9.0.4 - MIT Copyright (c) 2021 @@ -2798,7 +2798,7 @@ SOFTWARE. --------------------------------------------------------- -System.Data.OleDb 9.0.1 - MIT +System.Data.OleDb 9.0.4 - MIT Copyright (c) 2021 @@ -2941,7 +2941,7 @@ SOFTWARE. --------------------------------------------------------- -System.Diagnostics.DiagnosticSource 9.0.1 - MIT +System.Diagnostics.DiagnosticSource 9.0.4 - MIT Copyright (c) 2021 @@ -3031,7 +3031,7 @@ SOFTWARE. --------------------------------------------------------- -System.Diagnostics.EventLog 9.0.1 - MIT +System.Diagnostics.EventLog 9.0.4 - MIT Copyright (c) 2021 @@ -3121,7 +3121,7 @@ SOFTWARE. --------------------------------------------------------- -System.Diagnostics.PerformanceCounter 9.0.1 - MIT +System.Diagnostics.PerformanceCounter 9.0.4 - MIT Copyright (c) 2021 @@ -3211,7 +3211,7 @@ SOFTWARE. --------------------------------------------------------- -System.DirectoryServices 9.0.1 - MIT +System.DirectoryServices 9.0.4 - MIT Copyright (c) 2021 @@ -3301,7 +3301,7 @@ SOFTWARE. --------------------------------------------------------- -System.DirectoryServices.AccountManagement 9.0.1 - MIT +System.DirectoryServices.AccountManagement 9.0.4 - MIT Copyright (c) 2021 @@ -3391,7 +3391,7 @@ SOFTWARE. --------------------------------------------------------- -System.DirectoryServices.Protocols 9.0.1 - MIT +System.DirectoryServices.Protocols 9.0.4 - MIT Copyright (c) 2021 @@ -3481,7 +3481,7 @@ SOFTWARE. --------------------------------------------------------- -System.Drawing.Common 9.0.1 - MIT +System.Drawing.Common 9.0.4 - MIT (c) Microsoft Corporation @@ -3516,7 +3516,7 @@ SOFTWARE. --------------------------------------------------------- -System.IO.Packaging 9.0.1 - MIT +System.IO.Packaging 9.0.4 - MIT Copyright (c) 2021 @@ -3606,7 +3606,7 @@ SOFTWARE. --------------------------------------------------------- -System.IO.Ports 9.0.1 - MIT +System.IO.Ports 9.0.4 - MIT Copyright (c) 2021 @@ -3696,7 +3696,7 @@ SOFTWARE. --------------------------------------------------------- -System.Management 9.0.1 - MIT +System.Management 9.0.4 - MIT Copyright (c) 2021 @@ -3786,7 +3786,7 @@ SOFTWARE. --------------------------------------------------------- -System.Net.Http.WinHttpHandler 9.0.1 - MIT +System.Net.Http.WinHttpHandler 9.0.4 - MIT Copyright (c) 2021 @@ -3961,7 +3961,7 @@ SOFTWARE. --------------------------------------------------------- -System.Reflection.Context 9.0.1 - MIT +System.Reflection.Context 9.0.4 - MIT Copyright (c) 2021 @@ -4191,7 +4191,7 @@ SOFTWARE. --------------------------------------------------------- -System.Runtime.Caching 9.0.1 - MIT +System.Runtime.Caching 9.0.4 - MIT Copyright (c) 2021 @@ -4356,7 +4356,7 @@ SOFTWARE. --------------------------------------------------------- -System.Security.Cryptography.Pkcs 9.0.1 - MIT +System.Security.Cryptography.Pkcs 9.0.4 - MIT Copyright (c) 2021 @@ -4446,7 +4446,7 @@ SOFTWARE. --------------------------------------------------------- -System.Security.Cryptography.ProtectedData 9.0.1 - MIT +System.Security.Cryptography.ProtectedData 9.0.4 - MIT Copyright (c) 2021 @@ -4536,7 +4536,7 @@ SOFTWARE. --------------------------------------------------------- -System.Security.Cryptography.Xml 9.0.1 - MIT +System.Security.Cryptography.Xml 9.0.4 - MIT Copyright (c) 2021 @@ -4626,7 +4626,7 @@ SOFTWARE. --------------------------------------------------------- -System.Security.Permissions 9.0.1 - MIT +System.Security.Permissions 9.0.4 - MIT Copyright (c) 2021 @@ -4965,7 +4965,7 @@ SOFTWARE. --------------------------------------------------------- -System.ServiceModel.Syndication 9.0.1 - MIT +System.ServiceModel.Syndication 9.0.4 - MIT Copyright (c) 2021 @@ -5055,7 +5055,7 @@ SOFTWARE. --------------------------------------------------------- -System.ServiceProcess.ServiceController 9.0.1 - MIT +System.ServiceProcess.ServiceController 9.0.4 - MIT Copyright (c) 2021 @@ -5145,7 +5145,7 @@ SOFTWARE. --------------------------------------------------------- -System.Speech 9.0.1 - MIT +System.Speech 9.0.4 - MIT Copyright (c) 2021 @@ -5235,7 +5235,7 @@ SOFTWARE. --------------------------------------------------------- -System.Text.Encoding.CodePages 9.0.1 - MIT +System.Text.Encoding.CodePages 9.0.4 - MIT Copyright (c) 2021 @@ -5325,7 +5325,7 @@ SOFTWARE. --------------------------------------------------------- -System.Text.Encodings.Web 9.0.1 - MIT +System.Text.Encodings.Web 9.0.4 - MIT Copyright (c) 2021 @@ -5415,7 +5415,7 @@ SOFTWARE. --------------------------------------------------------- -System.Threading.AccessControl 9.0.1 - MIT +System.Threading.AccessControl 9.0.4 - MIT Copyright (c) 2021 @@ -5541,7 +5541,7 @@ SOFTWARE. --------------------------------------------------------- -System.Windows.Extensions 9.0.1 - MIT +System.Windows.Extensions 9.0.4 - MIT Copyright (c) 2021 diff --git a/build.psm1 b/build.psm1 index e21194c9af9..ffaf7b02f9d 100644 --- a/build.psm1 +++ b/build.psm1 @@ -179,6 +179,8 @@ function Get-EnvironmentInformation $environment += @{'IsUbuntu16' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '16.04'} $environment += @{'IsUbuntu18' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '18.04'} $environment += @{'IsUbuntu20' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '20.04'} + $environment += @{'IsUbuntu22' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '22.04'} + $environment += @{'IsUbuntu24' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '24.04'} $environment += @{'IsCentOS' = $LinuxInfo.ID -match 'centos' -and $LinuxInfo.VERSION_ID -match '7'} $environment += @{'IsFedora' = $LinuxInfo.ID -match 'fedora' -and $LinuxInfo.VERSION_ID -ge 24} $environment += @{'IsOpenSUSE' = $LinuxInfo.ID -match 'opensuse'} @@ -1229,6 +1231,7 @@ function Get-PesterTag { # testing PowerShell remote custom connections. function Publish-CustomConnectionTestModule { + Write-LogGroupStart -Title "Publish-CustomConnectionTestModule" $sourcePath = "${PSScriptRoot}/test/tools/NamedPipeConnection" $outPath = "${PSScriptRoot}/test/tools/NamedPipeConnection/out/Microsoft.PowerShell.NamedPipeConnection" $publishPath = "${PSScriptRoot}/test/tools/Modules" @@ -1253,6 +1256,8 @@ function Publish-CustomConnectionTestModule finally { Pop-Location } + + Write-LogGroupEnd -Title "Publish-CustomConnectionTestModule" } function Publish-PSTestTools { @@ -1262,6 +1267,7 @@ function Publish-PSTestTools { $runtime ) + Write-LogGroupStart -Title "Publish-PSTestTools" Find-Dotnet $tools = @( @@ -1333,6 +1339,7 @@ function Publish-PSTestTools { # Publish the Microsoft.PowerShell.NamedPipeConnection module Publish-CustomConnectionTestModule + Write-LogGroupEnd -Title "Publish-PSTestTools" } function Get-ExperimentalFeatureTests { @@ -1758,6 +1765,16 @@ function Publish-TestResults $resolvedPath = (Resolve-Path -Path $Path).ProviderPath Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$resolvedPath" + } elseif ($env:GITHUB_WORKFLOW -and $env:RUNNER_WORKSPACE) { + # In GitHub Actions + $destinationPath = Join-Path -Path $env:RUNNER_WORKSPACE -ChildPath 'testResults' + + # Create the folder if it does not exist + if (!(Test-Path -Path $destinationPath)) { + $null = New-Item -ItemType Directory -Path $destinationPath -Force + } + + Copy-Item -Path $Path -Destination $destinationPath -Force -Verbose } } @@ -1810,12 +1827,16 @@ function Show-PSPesterError throw 'Unknown Show-PSPester parameter set' } - Write-Log -isError -message ("Description: " + $description) - Write-Log -isError -message ("Name: " + $name) - Write-Log -isError -message "message:" - Write-Log -isError -message $message - Write-Log -isError -message "stack-trace:" - Write-Log -isError -message $stack_trace + # Empty line at the end is intentional formatting + Write-Log -isError -message @" +Description: $description +Name: $name +message: +$message +stack-trace: +$stack_trace + +"@ } @@ -1855,13 +1876,17 @@ function Test-XUnitTestResults $message = $failure.failure.message $stack_trace = $failure.failure.'stack-trace' - Write-Log -isError -message ("Description: " + $description) - Write-Log -isError -message ("Name: " + $name) - Write-Log -isError -message "message:" - Write-Log -isError -message $message - Write-Log -isError -message "stack-trace:" - Write-Log -isError -message $stack_trace - Write-Log -isError -message " " + # Empty line at the end is intentional formatting + Write-Log -isError -message @" + Description: $description + Name: $name + message: + $message + stack-trace: + $stack_trace + +"@ + } throw "$($results.assemblies.assembly.failed) tests failed" @@ -1897,7 +1922,8 @@ function Test-PSPesterResults $x = [xml](Get-Content -Raw $testResultsFile) if ([int]$x.'test-results'.failures -gt 0) { - Write-Log -isError -message "TEST FAILURES" + Write-LogGroupStart -Title 'TEST FAILURES' + # switch between methods, SelectNode is not available on dotnet core if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) { @@ -1911,6 +1937,8 @@ function Test-PSPesterResults { Show-PSPesterError -testFailure $testfail } + + Write-LogGroupEnd -Title 'TEST FAILURES' throw "$($x.'test-results'.failures) tests in $TestArea failed" } } @@ -1931,11 +1959,12 @@ function Test-PSPesterResults } elseif ($ResultObject.FailedCount -gt 0) { - Write-Log -isError -message 'TEST FAILURES' + Write-LogGroupStart -Title 'TEST FAILURES' $ResultObject.TestResult | Where-Object {$_.Passed -eq $false} | ForEach-Object { Show-PSPesterError -testFailureObject $_ } + Write-LogGroupEnd -Title 'TEST FAILURES' throw "$($ResultObject.FailedCount) tests in $TestArea failed" } @@ -2216,10 +2245,12 @@ function Start-PSBootstrap { # we currently pin dotnet-cli version, and will # update it when more stable version comes out. [string]$Version = $dotnetCLIRequiredVersion, - [switch]$Package, [switch]$NoSudo, [switch]$BuildLinuxArm, - [switch]$Force + [switch]$Force, + [Parameter(Mandatory = $true)] + [ValidateSet("Package", "DotNet", "Both")] + [string]$Scenario = "Package" ) Write-Log -message "Installing PowerShell build dependencies" @@ -2252,7 +2283,7 @@ function Start-PSBootstrap { elseif ($environment.IsUbuntu18) { $Deps += "libicu60"} # Packaging tools - if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev", "rpm", "g++", "make" } + if ($Scenario -eq 'Both' -or $Scenario -eq 'Package') { $Deps += "ruby-dev", "groff", "libffi-dev", "rpm", "g++", "make" } # Install dependencies # change the fontend from apt-get to noninteractive @@ -2276,7 +2307,7 @@ function Start-PSBootstrap { $Deps += "libicu", "openssl-libs" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" } + if ($Scenario -eq 'Both' -or $Scenario -eq 'Package') { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" } $PackageManager = Get-RedHatPackageManager @@ -2297,7 +2328,7 @@ function Start-PSBootstrap { $Deps += "wget" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel', "gcc" } + if ($Scenario -eq 'Both' -or $Scenario -eq 'Package') { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel', "gcc" } $PackageManager = "zypper --non-interactive install" $baseCommand = "$sudo $PackageManager" @@ -2337,7 +2368,7 @@ function Start-PSBootstrap { } # Install [fpm](https://github.com/jordansissel/fpm) - if ($Package) { + if ($Scenario -eq 'Both' -or $Scenario -eq 'Package') { Install-GlobalGem -Sudo $sudo -GemName "dotenv" -GemVersion "2.8.1" Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.16.3" Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.15.1" @@ -2345,42 +2376,45 @@ function Start-PSBootstrap { } } - Write-Verbose -Verbose "Calling Find-Dotnet from Start-PSBootstrap" + if ($Scenario -eq 'DotNet' -or $Scenario -eq 'Both') { - # Try to locate dotnet-SDK before installing it - Find-Dotnet + Write-Verbose -Verbose "Calling Find-Dotnet from Start-PSBootstrap" - Write-Verbose -Verbose "Back from calling Find-Dotnet from Start-PSBootstrap" + # Try to locate dotnet-SDK before installing it + Find-Dotnet - # Install dotnet-SDK - $dotNetExists = precheck 'dotnet' $null - $dotNetVersion = [string]::Empty - if($dotNetExists) { - $dotNetVersion = Find-RequiredSDK $dotnetCLIRequiredVersion - } + Write-Verbose -Verbose "Back from calling Find-Dotnet from Start-PSBootstrap" - if(!$dotNetExists -or $dotNetVersion -ne $dotnetCLIRequiredVersion -or $Force.IsPresent) { - if($Force.IsPresent) { - Write-Log -message "Installing dotnet due to -Force." - } - elseif(!$dotNetExists) { - Write-Log -message "dotnet not present. Installing dotnet." - } - else { - Write-Log -message "dotnet out of date ($dotNetVersion). Updating dotnet." + # Install dotnet-SDK + $dotNetExists = precheck 'dotnet' $null + $dotNetVersion = [string]::Empty + if($dotNetExists) { + $dotNetVersion = Find-RequiredSDK $dotnetCLIRequiredVersion } - $DotnetArguments = @{ Channel=$Channel; Version=$Version; NoSudo=$NoSudo } + if(!$dotNetExists -or $dotNetVersion -ne $dotnetCLIRequiredVersion -or $Force.IsPresent) { + if($Force.IsPresent) { + Write-Log -message "Installing dotnet due to -Force." + } + elseif(!$dotNetExists) { + Write-Log -message "dotnet not present. Installing dotnet." + } + else { + Write-Log -message "dotnet out of date ($dotNetVersion). Updating dotnet." + } - if ($dotnetAzureFeed) { - $null = $DotnetArguments.Add("AzureFeed", $dotnetAzureFeed) - $null = $DotnetArguments.Add("FeedCredential", $dotnetAzureFeedSecret) - } + $DotnetArguments = @{ Channel=$Channel; Version=$Version; NoSudo=$NoSudo } - Install-Dotnet @DotnetArguments - } - else { - Write-Log -message "dotnet is already installed. Skipping installation." + if ($dotnetAzureFeed) { + $null = $DotnetArguments.Add("AzureFeed", $dotnetAzureFeed) + $null = $DotnetArguments.Add("FeedCredential", $dotnetAzureFeedSecret) + } + + Install-Dotnet @DotnetArguments + } + else { + Write-Log -message "dotnet is already installed. Skipping installation." + } } # Install Windows dependencies if `-Package` or `-BuildWindowsNative` is specified @@ -2392,7 +2426,7 @@ function Start-PSBootstrap { $psInstallFile = [System.IO.Path]::Combine($PSScriptRoot, "tools", "install-powershell.ps1") & $psInstallFile -AddToPath } - if ($Package) { + if ($Scenario -eq 'Both' -or $Scenario -eq 'Package') { Import-Module "$PSScriptRoot\tools\wix\wix.psm1" $isArm64 = "$env:RUNTIME" -eq 'arm64' Install-Wix -arm64:$isArm64 @@ -2677,7 +2711,13 @@ function script:Write-Log ) if ($isError) { - Write-Host -Foreground Red $message + if ($env:GITHUB_WORKFLOW) { + # https://github.com/actions/toolkit/issues/193#issuecomment-605394935 + $escapedMessage = $message -replace "`n", "%0A" -replace "`r" + Write-Host "::error::${escapedMessage}" + } else { + Write-Host -Foreground Red $message + } } else { @@ -2686,6 +2726,59 @@ function script:Write-Log #reset colors for older package to at return to default after error message on a compilation error [console]::ResetColor() } + +function script:Write-LogGroup { + param + ( + [Parameter(Position = 0, Mandatory)] + [ValidateNotNullOrEmpty()] + [string[]] $Message, + [Parameter(Mandatory)] + [string] $Title + ) + + + Write-LogGroupStart -Title $Title + + foreach ($line in $Message) { + Write-Log -Message $line + } + + Write-LogGroupEnd -Title $Title +} + +$script:logGroupColor = [System.ConsoleColor]::Cyan + +function script:Write-LogGroupStart { + param + ( + [Parameter(Mandatory)] + [string] $Title + ) + + if ($env:GITHUB_WORKFLOW) { + Write-Host "::group::${Title}" + } + else { + Write-Host -ForegroundColor $script:logGroupColor "=== BEGIN: $Title ===" + } +} + +function script:Write-LogGroupEnd { + param + ( + [Parameter(Mandatory)] + [string] $Title + ) + + if ($env:GITHUB_WORKFLOW) { + Write-Host "::endgroup::" + } + else { + Write-Host -ForegroundColor $script:logGroupColor "==== END: $Title ====" + } +} + function script:precheck([string]$command, [string]$missedMessage) { $c = Get-Command $command -ErrorAction Ignore if (-not $c) { @@ -3583,6 +3676,14 @@ function New-NugetConfigFile { $content += $newLine + $nugetConfigFooterTemplate Set-Content -Path (Join-Path $Destination 'nuget.config') -Value $content -Force + + # Set the nuget.config file to be skipped by git + push-location $Destination + try { + git update-index --skip-worktree (Join-Path $Destination 'nuget.config') + } finally { + pop-location + } } function Clear-PipelineNugetAuthentication { @@ -3604,22 +3705,50 @@ function Set-PipelineNugetAuthentication { function Set-CorrectLocale { + Write-LogGroupStart -Title "Set-CorrectLocale" + if (-not $IsLinux) { + Write-LogGroupEnd -Title "Set-CorrectLocale" return } $environment = Get-EnvironmentInformation - if ($environment.IsUbuntu -and $environment.IsUbuntu20) - { + if ($environment.IsUbuntu16 -or $environment.IsUbuntu18) { + Write-Verbose -Message "Don't set locale before Ubuntu 20" -Verbose + Write-LogGroupEnd -Title "Set-CorrectLocale" + Write-Locale + return + } + + if ($environment.IsUbuntu) { + Write-Log -Message "Setting locale to en_US.UTF-8" $env:LC_ALL = 'en_US.UTF-8' $env:LANG = 'en_US.UTF-8' sudo locale-gen $env:LANG - sudo update-locale + if ($environment.IsUbuntu20) { + Write-Log -Message "Updating locale for Ubuntu 20" + sudo update-locale + } else { + Write-Log -Message "Updating locale for Ubuntu 22 and newer" + sudo update-locale LANG=$env:LANG LC_ALL=$env:LC_ALL + } + } + + Write-LogGroupEnd -Title "Set-CorrectLocale" + Write-Locale + +} + +function Write-Locale { + if (-not $IsLinux -and -not $IsMacOS) { + Write-Verbose -Message "only supported on Linux and macOS" -Verbose + return } # Output the locale to log it - locale + $localOutput = & locale + Write-LogGroup -Title "Capture Locale" -Message $localOutput } function Install-AzCopy { diff --git a/global.json b/global.json index ee2876ea570..e4da652c648 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "9.0.102" + "version": "9.0.203" } } diff --git a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj index d17d99b190d..4952c8f6ad4 100644 --- a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj +++ b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj @@ -7,9 +7,11 @@ - + + + - + diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index a4cb9ad9dfb..4fe6ed61803 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,8 +47,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index bc15337da1f..1cc347541a2 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + @@ -41,8 +41,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/AddType.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/AddType.cs index 81ca82cb3c3..7dc0a9c3556 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/AddType.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/AddType.cs @@ -684,6 +684,7 @@ private void LoadAssemblies(IEnumerable assemblies) { // CoreCLR doesn't allow re-load TPA assemblies with different API (i.e. we load them by name and now want to load by path). // LoadAssemblyHelper helps us avoid re-loading them, if they already loaded. + // codeql[cs/dll-injection-remote] - This is expected PowerShell behavior and integral to the purpose of the class. It allows users to load any C# dependencies they need for their PowerShell application and add other types they require. Assembly assembly = LoadAssemblyHelper(assemblyName) ?? Assembly.LoadFrom(ResolveAssemblyName(assemblyName, false)); if (PassThru) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 886c04919b6..810b54a8391 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1771,6 +1771,7 @@ private static StringContent GetMultipartStringContent(object fieldName, object ContentDispositionHeaderValue contentDisposition = new("form-data"); contentDisposition.Name = LanguagePrimitives.ConvertTo(fieldName); + // codeql[cs/information-exposure-through-exception] - PowerShell is an on-premise product, meaning local users would already have access to the binaries and stack traces. Therefore, the information would not be exposed in the same way it would be for an ASP .NET service. StringContent result = new(LanguagePrimitives.ConvertTo(fieldValue)); result.Headers.ContentDisposition = contentDisposition; diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index bd8754bff4d..e5f5b849645 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 75492f49c6c..1d1fc2f1291 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -17,55 +17,55 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index c9d448a5091..db972fa45c8 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -7,11 +7,11 @@ - + - + diff --git a/src/Modules/PSGalleryModules.csproj b/src/Modules/PSGalleryModules.csproj index 5f9f89a4563..cb799b441fc 100644 --- a/src/Modules/PSGalleryModules.csproj +++ b/src/Modules/PSGalleryModules.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index d0e81fddb6d..afc4c178cb3 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -32,25 +32,25 @@ - - - - - - + + + + + + - + - - - - + + + + - - + + diff --git a/src/System.Management.Automation/engine/ExecutionContext.cs b/src/System.Management.Automation/engine/ExecutionContext.cs index 56f64c1a5c2..e0e078346e9 100644 --- a/src/System.Management.Automation/engine/ExecutionContext.cs +++ b/src/System.Management.Automation/engine/ExecutionContext.cs @@ -1385,6 +1385,7 @@ private static Assembly LoadAssembly(string name, string filePath, out Exception { try { + // codeql[cs/dll-injection-remote] - The dll is loaded during the initial state setup, which is expected behavior. This allows users hosting PowerShell to load additional C# types to enable their specific scenarios. loadedAssembly = Assembly.LoadFrom(filePath); return loadedAssembly; } diff --git a/src/System.Management.Automation/security/wldpNativeMethods.cs b/src/System.Management.Automation/security/wldpNativeMethods.cs index a59f37c0a8f..ab49f927614 100644 --- a/src/System.Management.Automation/security/wldpNativeMethods.cs +++ b/src/System.Management.Automation/security/wldpNativeMethods.cs @@ -6,6 +6,7 @@ // #if !UNIX +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Management.Automation.Internal; using System.Management.Automation.Runspaces; @@ -148,7 +149,7 @@ public static SystemEnforcementMode GetSystemLockdownPolicy() { lock (s_systemLockdownPolicyLock) { - s_systemLockdownPolicy = GetDebugLockdownPolicy(path: null); + s_systemLockdownPolicy = GetDebugLockdownPolicy(path: null, out _); } } @@ -172,93 +173,89 @@ public static SystemScriptFileEnforcement GetFilePolicyEnforcement( System.IO.FileStream fileStream) { SafeHandle fileHandle = fileStream.SafeFileHandle; - var systemLockdownPolicy = SystemPolicy.GetSystemLockdownPolicy(); + SystemEnforcementMode systemLockdownPolicy = GetSystemLockdownPolicy(); // First check latest WDAC APIs if available. - // Revert to legacy APIs if system policy is in AUDIT mode or debug hook is in effect. - Exception errorException = null; - if (s_wldpCanExecuteAvailable && systemLockdownPolicy == SystemEnforcementMode.Enforce) + if (systemLockdownPolicy is SystemEnforcementMode.Enforce + && s_wldpCanExecuteAvailable + && TryGetWldpCanExecuteFileResult(filePath, fileHandle, out SystemScriptFileEnforcement wldpFilePolicy)) { - try - { - string fileName = System.IO.Path.GetFileNameWithoutExtension(filePath); - string auditMsg = $"PowerShell ExternalScriptInfo reading file: {fileName}"; + return GetLockdownPolicy(filePath, fileHandle, wldpFilePolicy); + } - int hr = WldpNativeMethods.WldpCanExecuteFile( - host: PowerShellHost, - options: WLDP_EXECUTION_EVALUATION_OPTIONS.WLDP_EXECUTION_EVALUATION_OPTION_NONE, - fileHandle: fileHandle.DangerousGetHandle(), - auditInfo: auditMsg, - result: out WLDP_EXECUTION_POLICY canExecuteResult); + // Failed to invoke WldpCanExecuteFile, revert to legacy APIs. + if (systemLockdownPolicy is SystemEnforcementMode.None) + { + return SystemScriptFileEnforcement.None; + } - PSEtwLog.LogWDACQueryEvent("WldpCanExecuteFile", filePath, hr, (int)canExecuteResult); + // WldpCanExecuteFile was invoked successfully so we can skip running + // legacy WDAC APIs. AppLocker must still be checked in case it is more + // strict than the current WDAC policy. + return GetLockdownPolicy(filePath, fileHandle, canExecuteResult: null); + } - if (hr >= 0) - { - switch (canExecuteResult) - { - case WLDP_EXECUTION_POLICY.WLDP_CAN_EXECUTE_ALLOWED: - return SystemScriptFileEnforcement.Allow; + private static SystemScriptFileEnforcement ConvertToModernFileEnforcement(SystemEnforcementMode legacyMode) + { + return legacyMode switch + { + SystemEnforcementMode.None => SystemScriptFileEnforcement.Allow, + SystemEnforcementMode.Audit => SystemScriptFileEnforcement.AllowConstrainedAudit, + SystemEnforcementMode.Enforce => SystemScriptFileEnforcement.AllowConstrained, + _ => SystemScriptFileEnforcement.Block, + }; + } - case WLDP_EXECUTION_POLICY.WLDP_CAN_EXECUTE_BLOCKED: - return SystemScriptFileEnforcement.Block; + private static bool TryGetWldpCanExecuteFileResult(string filePath, SafeHandle fileHandle, out SystemScriptFileEnforcement result) + { + try + { + string fileName = System.IO.Path.GetFileNameWithoutExtension(filePath); + string auditMsg = $"PowerShell ExternalScriptInfo reading file: {fileName}"; - case WLDP_EXECUTION_POLICY.WLDP_CAN_EXECUTE_REQUIRE_SANDBOX: - return SystemScriptFileEnforcement.AllowConstrained; + int hr = WldpNativeMethods.WldpCanExecuteFile( + host: PowerShellHost, + options: WLDP_EXECUTION_EVALUATION_OPTIONS.WLDP_EXECUTION_EVALUATION_OPTION_NONE, + fileHandle: fileHandle.DangerousGetHandle(), + auditInfo: auditMsg, + result: out WLDP_EXECUTION_POLICY canExecuteResult); - default: - // Fall through to legacy system policy checks. - System.Diagnostics.Debug.Assert(false, $"Unknown execution policy returned from WldCanExecute: {canExecuteResult}"); - break; - } - } + PSEtwLog.LogWDACQueryEvent("WldpCanExecuteFile", filePath, hr, (int)canExecuteResult); - // If HResult is unsuccessful (such as E_NOTIMPL (0x80004001)), fall through to legacy system checks. - } - catch (DllNotFoundException ex) - { - // Fall back to legacy system policy checks. - s_wldpCanExecuteAvailable = false; - errorException = ex; - } - catch (EntryPointNotFoundException ex) + if (hr >= 0) { - // Fall back to legacy system policy checks. - s_wldpCanExecuteAvailable = false; - errorException = ex; + switch (canExecuteResult) + { + case WLDP_EXECUTION_POLICY.WLDP_CAN_EXECUTE_ALLOWED: + result = SystemScriptFileEnforcement.Allow; + return true; + + case WLDP_EXECUTION_POLICY.WLDP_CAN_EXECUTE_BLOCKED: + result = SystemScriptFileEnforcement.Block; + return true; + + case WLDP_EXECUTION_POLICY.WLDP_CAN_EXECUTE_REQUIRE_SANDBOX: + result = SystemScriptFileEnforcement.AllowConstrained; + return true; + + default: + // Fall through to legacy system policy checks. + Debug.Assert(false, $"Unknown policy result returned from WldCanExecute: {canExecuteResult}"); + break; + } } - if (errorException != null) - { - PSEtwLog.LogWDACQueryEvent("WldpCanExecuteFile_Failed", filePath, errorException.HResult, 0); - } + // If HResult is unsuccessful (such as E_NOTIMPL (0x80004001)), fall through to legacy system checks. } - - // Original (legacy) WDAC and AppLocker system checks. - if (systemLockdownPolicy == SystemEnforcementMode.None) + catch (Exception ex) when (ex is DllNotFoundException or EntryPointNotFoundException) { - return SystemScriptFileEnforcement.None; + // Fall back to legacy system policy checks. + s_wldpCanExecuteAvailable = false; + PSEtwLog.LogWDACQueryEvent("WldpCanExecuteFile_Failed", filePath, ex.HResult, 0); } - // Check policy for file. - switch (SystemPolicy.GetLockdownPolicy(filePath, fileHandle)) - { - case SystemEnforcementMode.Enforce: - // File is not allowed by policy enforcement and must run in CL mode. - return SystemScriptFileEnforcement.AllowConstrained; - - case SystemEnforcementMode.Audit: - // File is allowed but would be run in CL mode if policy was enforced and not audit. - return SystemScriptFileEnforcement.AllowConstrainedAudit; - - case SystemEnforcementMode.None: - // No restrictions, file will run in FL mode. - return SystemScriptFileEnforcement.Allow; - - default: - System.Diagnostics.Debug.Assert(false, "GetFilePolicyEnforcement: Unknown SystemEnforcementMode."); - return SystemScriptFileEnforcement.Block; - } + result = default; + return false; } /// @@ -267,9 +264,32 @@ public static SystemScriptFileEnforcement GetFilePolicyEnforcement( /// An EnforcementMode that describes policy. public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle handle) { + SystemScriptFileEnforcement modernMode = GetLockdownPolicy(path, handle, canExecuteResult: null); + Debug.Assert( + modernMode is not SystemScriptFileEnforcement.Block, + "Block should never be converted to legacy file enforcement."); + + return modernMode switch + { + SystemScriptFileEnforcement.Block => SystemEnforcementMode.Enforce, + SystemScriptFileEnforcement.AllowConstrained => SystemEnforcementMode.Enforce, + SystemScriptFileEnforcement.AllowConstrainedAudit => SystemEnforcementMode.Audit, + SystemScriptFileEnforcement.Allow => SystemEnforcementMode.None, + SystemScriptFileEnforcement.None => SystemEnforcementMode.None, + _ => throw new ArgumentOutOfRangeException(nameof(modernMode)), + }; + } + + private static SystemScriptFileEnforcement GetLockdownPolicy( + string path, + SafeHandle handle, + SystemScriptFileEnforcement? canExecuteResult) + { + SystemScriptFileEnforcement wldpFilePolicy = canExecuteResult + ?? ConvertToModernFileEnforcement(GetWldpPolicy(path, handle)); + // Check the WLDP File policy via API - var wldpFilePolicy = GetWldpPolicy(path, handle); - if (wldpFilePolicy == SystemEnforcementMode.Enforce) + if (wldpFilePolicy is SystemScriptFileEnforcement.Block or SystemScriptFileEnforcement.AllowConstrained) { return wldpFilePolicy; } @@ -281,29 +301,28 @@ public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle ha var appLockerFilePolicy = GetAppLockerPolicy(path, handle); if (appLockerFilePolicy == SystemEnforcementMode.Enforce) { - return appLockerFilePolicy; + return ConvertToModernFileEnforcement(appLockerFilePolicy); } // At this point, LockdownPolicy = Audit or Allowed. // If there was a WLDP policy, but WLDP didn't block it, // then it was explicitly allowed. Therefore, return the result for the file. - SystemEnforcementMode systemWldpPolicy = s_cachedWldpSystemPolicy.GetValueOrDefault(SystemEnforcementMode.None); - if ((systemWldpPolicy == SystemEnforcementMode.Audit) || - (systemWldpPolicy == SystemEnforcementMode.Enforce)) + if (s_cachedWldpSystemPolicy is SystemEnforcementMode.Audit or SystemEnforcementMode.Enforce + || wldpFilePolicy is SystemScriptFileEnforcement.AllowConstrainedAudit) { return wldpFilePolicy; } // If there was a system-wide AppLocker policy, but AppLocker didn't block it, // then return AppLocker's status. - if (s_cachedSaferSystemPolicy.GetValueOrDefault(SaferPolicy.Allowed) == - SaferPolicy.Disallowed) + if (s_cachedSaferSystemPolicy is SaferPolicy.Disallowed) { - return appLockerFilePolicy; + return ConvertToModernFileEnforcement(appLockerFilePolicy); } // If it's not set to 'Enforce' by the platform, allow debug overrides - return GetDebugLockdownPolicy(path); + GetDebugLockdownPolicy(path, out SystemScriptFileEnforcement debugPolicy); + return debugPolicy; } [SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods", @@ -558,7 +577,7 @@ private static SaferPolicy TestSaferPolicy(string testPathScript, string testPat return result; } - private static SystemEnforcementMode GetDebugLockdownPolicy(string path) + private static SystemEnforcementMode GetDebugLockdownPolicy(string path, out SystemScriptFileEnforcement modernEnforcement) { s_allowDebugOverridePolicy = true; @@ -569,10 +588,19 @@ private static SystemEnforcementMode GetDebugLockdownPolicy(string path) // check so that we can actually put it in the filename during testing. if (path.Contains("System32", StringComparison.OrdinalIgnoreCase)) { + modernEnforcement = SystemScriptFileEnforcement.Allow; return SystemEnforcementMode.None; } // No explicit debug allowance for the file, so return the system policy if there is one. + modernEnforcement = s_systemLockdownPolicy switch + { + SystemEnforcementMode.Enforce => SystemScriptFileEnforcement.AllowConstrained, + SystemEnforcementMode.Audit => SystemScriptFileEnforcement.AllowConstrainedAudit, + SystemEnforcementMode.None => SystemScriptFileEnforcement.None, + _ => SystemScriptFileEnforcement.None, + }; + return s_systemLockdownPolicy.GetValueOrDefault(SystemEnforcementMode.None); } @@ -582,10 +610,13 @@ private static SystemEnforcementMode GetDebugLockdownPolicy(string path) if (result != null) { pdwLockdownState = LanguagePrimitives.ConvertTo(result); - return GetLockdownPolicyForResult(pdwLockdownState); + SystemEnforcementMode policy = GetLockdownPolicyForResult(pdwLockdownState); + modernEnforcement = ConvertToModernFileEnforcement(policy); + return policy; } // If the system-wide debug policy had no preference, then there is no enforcement. + modernEnforcement = SystemScriptFileEnforcement.None; return SystemEnforcementMode.None; } diff --git a/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index 427245af6d8..6f6ef0087ab 100644 --- a/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj b/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj index f4874cfa6d7..164023a0f62 100644 --- a/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj +++ b/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj b/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj index 8bf1139db7d..aa15f08cb56 100644 --- a/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj +++ b/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj @@ -17,21 +17,21 @@ - + - - - - - - - + + + + + + + - - - - - + + + + + diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index 68452edfdb5..31a95e0b875 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -15,56 +15,56 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + - + diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj index 7117e235557..b609555c9ad 100644 --- a/test/tools/WebListener/WebListener.csproj +++ b/test/tools/WebListener/WebListener.csproj @@ -7,9 +7,9 @@ - - + + - + diff --git a/tools/cgmanifest.json b/tools/cgmanifest.json index bb4fd3edb1c..e64601d8eaf 100644 --- a/tools/cgmanifest.json +++ b/tools/cgmanifest.json @@ -115,17 +115,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Extensions.ObjectPool", - "Version": "8.0.12" - } - }, - "DevelopmentDependency": false - }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "Microsoft.Management.Infrastructure.Runtime.Unix", - "Version": "3.0.0" + "Version": "8.0.15" } }, "DevelopmentDependency": false @@ -140,16 +130,6 @@ }, "DevelopmentDependency": true }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "Microsoft.Management.Infrastructure", - "Version": "3.0.0" - } - }, - "DevelopmentDependency": false - }, { "Component": { "Type": "nuget", @@ -160,22 +140,12 @@ }, "DevelopmentDependency": false }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "Microsoft.PowerShell.Native", - "Version": "7.4.0" - } - }, - "DevelopmentDependency": false - }, { "Component": { "Type": "nuget", "Nuget": { "Name": "Microsoft.Security.Extensions", - "Version": "1.3.0" + "Version": "1.4.0" } }, "DevelopmentDependency": false @@ -185,7 +155,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.Registry.AccessControl", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -205,7 +175,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.SystemEvents", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -215,7 +185,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Windows.Compatibility", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -235,7 +205,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-arm.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -245,7 +215,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -255,7 +225,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-x64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -265,7 +235,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-x86.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -275,7 +245,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-arm.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -285,7 +255,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -295,7 +265,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-bionic-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -305,7 +275,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-bionic-x64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -315,7 +285,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-arm.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -325,7 +295,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -335,7 +305,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-x64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -345,7 +315,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-x64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -355,7 +325,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.maccatalyst-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -365,7 +335,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.maccatalyst-x64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -385,7 +355,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -395,7 +365,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.osx-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -405,7 +375,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.osx-x64.runtime.native.System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -465,7 +435,7 @@ "Type": "nuget", "Nuget": { "Name": "System.CodeDom", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -485,7 +455,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ComponentModel.Composition.Registration", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -495,7 +465,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ComponentModel.Composition", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -505,7 +475,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Configuration.ConfigurationManager", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -515,7 +485,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.Odbc", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -525,7 +495,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.OleDb", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -545,7 +515,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.DiagnosticSource", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -555,7 +525,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.EventLog", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -565,7 +535,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.PerformanceCounter", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -575,7 +545,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.AccountManagement", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -585,7 +555,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.Protocols", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -595,7 +565,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -605,7 +575,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Drawing.Common", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -615,7 +585,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Packaging", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -625,7 +595,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Ports", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -635,7 +605,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Management", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -645,7 +615,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Net.Http.WinHttpHandler", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -675,7 +645,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Reflection.Context", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -705,7 +675,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Runtime.Caching", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -725,7 +695,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Pkcs", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -735,7 +705,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.ProtectedData", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -745,7 +715,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Xml", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -755,7 +725,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Permissions", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -825,7 +795,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceModel.Syndication", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -835,7 +805,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceProcess.ServiceController", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -845,7 +815,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Speech", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -855,7 +825,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Text.Encoding.CodePages", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -865,7 +835,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Text.Encodings.Web", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -875,7 +845,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Threading.AccessControl", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false @@ -895,7 +865,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Windows.Extensions", - "Version": "9.0.1" + "Version": "9.0.4" } }, "DevelopmentDependency": false diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 6628d54e043..317f05effd0 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -17,8 +17,15 @@ if(Test-Path $dotNetPath) # import build into the global scope so it can be used by packaging # argumentList $true says ignore tha we may not be able to build -Import-Module (Join-Path $repoRoot 'build.psm1') -Verbose -Scope Global -ArgumentList $true -Import-Module (Join-Path $repoRoot 'tools\packaging') -Verbose -Scope Global +Write-Verbose "Importing build.psm1" -Verbose +Import-Module (Join-Path $repoRoot 'build.psm1') -Scope Global -ArgumentList $true +$buildCommands = Get-Command -Module build +Write-Verbose "Imported build.psm1 commands: $($buildCommands.Count)" -Verbose + +Write-Verbose "Importing packaging.psm1" -Verbose +Import-Module (Join-Path $repoRoot 'tools\packaging') -Scope Global +$packagingCommands = Get-Command -Module packaging +Write-Verbose "Imported packaging.psm1 commands: $($packagingCommands.Count)" -Verbose # import the windows specific functcion only in Windows PowerShell or on Windows if($PSVersionTable.PSEdition -eq 'Desktop' -or $IsWindows) @@ -181,8 +188,6 @@ function Invoke-CIInstall } Set-BuildVariable -Name TestPassed -Value False - Write-Verbose -Verbose -Message "Calling Start-PSBootstrap from Invoke-CIInstall" - Start-PSBootstrap } function Invoke-CIxUnit @@ -224,9 +229,12 @@ function Invoke-CITest [string] $Purpose, [ValidateSet('CI', 'Others')] [string] $TagSet, - [string] $TitlePrefix + [string] $TitlePrefix, + [string] $OutputFormat = "NUnitXml" ) + Write-Verbose -Verbose "CI test: OutputFormat: $OutputFormat" + # Set locale correctly for Linux CIs Set-CorrectLocale @@ -249,7 +257,7 @@ function Invoke-CITest if($IsLinux -or $IsMacOS) { - return Invoke-LinuxTestsCore -Purpose $Purpose -ExcludeTag $ExcludeTag -TagSet $TagSet -TitlePrefix $TitlePrefix + return Invoke-LinuxTestsCore -Purpose $Purpose -ExcludeTag $ExcludeTag -TagSet $TagSet -TitlePrefix $TitlePrefix -OutputFormat $OutputFormat } # CoreCLR @@ -281,12 +289,14 @@ function Invoke-CITest Terse = $true Tag = @() ExcludeTag = $ExcludeTag + 'RequireAdminOnWindows' + OutputFormat = $OutputFormat } $title = "Pester Unelevated - $TagSet" if ($TitlePrefix) { $title = "$TitlePrefix - $title" } + Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)" Start-PSPester @arguments -Title $title # Fail the build, if tests failed @@ -314,7 +324,10 @@ function Invoke-CITest if ($TitlePrefix) { $title = "$TitlePrefix - $title" } - Start-PSPester @arguments -Title $title + + # We just built the test tools, we don't need to rebuild them + Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)" + Start-PSPester @arguments -Title $title -SkipTestToolBuild # Fail the build, if tests failed Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile @@ -328,12 +341,15 @@ function Invoke-CITest OutputFile = $testResultsAdminFile Tag = @('RequireAdminOnWindows') ExcludeTag = $ExcludeTag + OutputFormat = $OutputFormat } $title = "Pester Elevated - $TagSet" if ($TitlePrefix) { $title = "$TitlePrefix - $title" } + + Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)" Start-PSPester @arguments -Title $title # Fail the build, if tests failed @@ -364,7 +380,10 @@ function Invoke-CITest if ($TitlePrefix) { $title = "$TitlePrefix - $title" } - Start-PSPester @arguments -Title $title + + Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)" + # We just built the test tools, we don't need to rebuild them + Start-PSPester @arguments -Title $title -SkipTestToolBuild # Fail the build, if tests failed Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile @@ -381,8 +400,6 @@ function New-CodeCoverageAndTestPackage if (Test-DailyBuild) { - Start-PSBootstrap -Verbose - Start-PSBuild -Configuration 'CodeCoverage' -Clean $codeCoverageOutput = Split-Path -Parent (Get-PSOutput) @@ -437,6 +454,18 @@ function Push-Artifact if ($env:TF_BUILD) { # In Azure DevOps Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$Path" + } elseif ($env:GITHUB_WORKFLOW -and $env:RUNNER_WORKSPACE) { + # In GitHub Actions + $destinationPath = Join-Path -Path $env:RUNNER_WORKSPACE -ChildPath $artifactName + + # Create the folder if it does not exist + if (!(Test-Path -Path $destinationPath)) { + $null = New-Item -ItemType Directory -Path $destinationPath -Force + } + + Copy-Item -Path $Path -Destination $destinationPath -Force -Verbose + } else { + Write-Warning "Push-Artifact is not supported in this environment." } } @@ -658,7 +687,7 @@ function Invoke-BootstrapStage Write-Log -Message "Executing ci.psm1 Bootstrap Stage" # Make sure we have all the tags Sync-PSTags -AddRemoteIfMissing - Start-PSBootstrap -Package:$createPackages + Start-PSBootstrap -Scenario Package:$createPackages } # Run pester tests for Linux and macOS @@ -670,7 +699,8 @@ function Invoke-LinuxTestsCore [string] $Purpose = 'All', [string[]] $ExcludeTag = @('Slow', 'Feature', 'Scenario'), [string] $TagSet = 'CI', - [string] $TitlePrefix + [string] $TitlePrefix, + [string] $OutputFormat = "NUnitXml" ) $output = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions)) @@ -683,12 +713,13 @@ function Invoke-LinuxTestsCore $sudoResultsWithExpFeatures = $null $noSudoPesterParam = @{ - 'BinDir' = $output - 'PassThru' = $true - 'Terse' = $true - 'Tag' = @() - 'ExcludeTag' = $testExcludeTag - 'OutputFile' = $testResultsNoSudo + 'BinDir' = $output + 'PassThru' = $true + 'Terse' = $true + 'Tag' = @() + 'ExcludeTag' = $testExcludeTag + 'OutputFile' = $testResultsNoSudo + 'OutputFormat' = $OutputFormat } # Get the experimental feature names and the tests associated with them @@ -726,7 +757,7 @@ function Invoke-LinuxTestsCore if ($TitlePrefix) { $title = "$TitlePrefix - $title" } - $passThruResult = Start-PSPester @noSudoPesterParam -Title $title + $passThruResult = Start-PSPester @noSudoPesterParam -Title $title -SkipTestToolBuild $noSudoResultsWithExpFeatures += $passThruResult } @@ -741,6 +772,7 @@ function Invoke-LinuxTestsCore $sudoPesterParam['ExcludeTag'] = $ExcludeTag $sudoPesterParam['Sudo'] = $true $sudoPesterParam['OutputFile'] = $testResultsSudo + $sudoPesterParam['OutputFormat'] = $OutputFormat $title = "Pester Sudo - $TagSet" if ($TitlePrefix) { @@ -773,7 +805,9 @@ function Invoke-LinuxTestsCore if ($TitlePrefix) { $title = "$TitlePrefix - $title" } - $passThruResult = Start-PSPester @sudoPesterParam -Title $title + + # We just built the test tools for the main test run, we don't need to rebuild them + $passThruResult = Start-PSPester @sudoPesterParam -Title $title -SkipTestToolBuild $sudoResultsWithExpFeatures += $passThruResult } diff --git a/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 b/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 index 2a9434c9cbe..4d874402977 100644 --- a/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 +++ b/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 @@ -27,7 +27,7 @@ function Start-ClearlyDefinedHarvest { $coordinates = Get-ClearlyDefinedCoordinates @PSBoundParameters $body = @{tool='package';coordinates=$coordinates} | convertto-json Write-Verbose $body -Verbose - (Invoke-WebRequest -Method Post -Uri 'https://api.clearlydefined.io/harvest' -Body $body -ContentType 'application/json').Content + (Invoke-WebRequest -Method Post -Uri 'https://api.clearlydefined.io/harvest' -Body $body -ContentType 'application/json' -MaximumRetryCount 5 -RetryIntervalSec 60 -Verbose).Content } } @@ -117,7 +117,7 @@ Function Get-ClearlyDefinedData { continue } - Invoke-RestMethod -Uri "https://api.clearlydefined.io/definitions/$coordinates" | ForEach-Object { + Invoke-RestMethod -Uri "https://api.clearlydefined.io/definitions/$coordinates" -MaximumRetryCount 5 -RetryIntervalSec 60 | ForEach-Object { [bool] $harvested = if ($_.licensed.declared) { $true } else { $false } Add-Member -NotePropertyName cachedTime -NotePropertyValue (get-date) -InputObject $_ -PassThru | Add-Member -NotePropertyName harvested -NotePropertyValue $harvested -PassThru if ($_.harvested) { diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index 490edebb81b..3200bdeeaaf 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -203,28 +203,34 @@ function Get-CGRegistrations { "alpine-.*" { $folder = $unixProjectName $target = "$dotnetTargetName|$Runtime" + $neutralTarget = "$dotnetTargetName" } "linux-.*" { $folder = $unixProjectName $target = "$dotnetTargetName|$Runtime" + $neutralTarget = "$dotnetTargetName" } "osx-.*" { $folder = $unixProjectName $target = "$dotnetTargetName|$Runtime" + $neutralTarget = "$dotnetTargetName" } "win-x*" { $sdkToUse = $winDesktopSdk $folder = $windowsProjectName $target = "$dotnetTargetNameWin7|$Runtime" + $neutralTarget = "$dotnetTargetNameWin7" } "win-.*" { $folder = $windowsProjectName $target = "$dotnetTargetNameWin7|$Runtime" + $neutralTarget = "$dotnetTargetNameWin7" } "modules" { $folder = "modules" $actualRuntime = 'linux-x64' $target = "$dotnetTargetName|$actualRuntime" + $neutralTarget = "$dotnetTargetName" } Default { throw "Invalid runtime name: $Runtime" @@ -241,6 +247,7 @@ function Get-CGRegistrations { $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder try { $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { $_.Type -eq 'package' } | select-object -ExpandProperty name + $targets += Get-ChildItem -Path "${folder}:/targets/$neutralTarget" -ErrorAction Stop | Where-Object { $_.Type -eq 'project' } | select-object -ExpandProperty name } catch { Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose throw @@ -250,27 +257,53 @@ function Get-CGRegistrations { Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive } + # Name to skip for TPN generation + $skipNames = @( + "Microsoft.PowerShell.Native" + "Microsoft.Management.Infrastructure.Runtime.Unix" + "Microsoft.Management.Infrastructure" + "Microsoft.PowerShell.Commands.Diagnostics" + "Microsoft.PowerShell.Commands.Management" + "Microsoft.PowerShell.Commands.Utility" + "Microsoft.PowerShell.ConsoleHost" + "Microsoft.PowerShell.SDK" + "Microsoft.PowerShell.Security" + "Microsoft.Management.Infrastructure.CimCmdlets" + "Microsoft.WSMan.Management" + "Microsoft.WSMan.Runtime" + "System.Management.Automation" + "Microsoft.PowerShell.GraphicalHost" + "Microsoft.PowerShell.CoreCLR.Eventing" + ) + + Write-Verbose "Found $($targets.Count) targets to process..." -Verbose $targets | ForEach-Object { $target = $_ $parts = ($target -split '\|') $name = $parts[0] - $targetVersion = $parts[1] - $publicVersion = Get-NuGetPublicVersion -Name $name -Version $targetVersion - - # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR - # the exisitng CG contains the registration, because if the existing CG contains the registration, - # that might be the only reason it is in the TPN. - if (!$RegistrationTable.ContainsKey($target)) { - $DevelopmentDependency = $false - if (!$existingRegistrationTable.ContainsKey($name) -or $existingRegistrationTable.$name.Component.Version() -ne $publicVersion) { - $registrationChanged = $true - } - if ($existingRegistrationTable.ContainsKey($name) -and $existingRegistrationTable.$name.DevelopmentDependency) { - $DevelopmentDependency = $true - } - $registration = New-NugetComponent -Name $name -Version $publicVersion -DevelopmentDependency:$DevelopmentDependency - $RegistrationTable.Add($target, $registration) + if ($name -in $skipNames) { + Write-Verbose "Skipping $name..." + + } else { + $targetVersion = $parts[1] + $publicVersion = Get-NuGetPublicVersion -Name $name -Version $targetVersion + + # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR + # the exisitng CG contains the registration, because if the existing CG contains the registration, + # that might be the only reason it is in the TPN. + if (!$RegistrationTable.ContainsKey($target)) { + $DevelopmentDependency = $false + if (!$existingRegistrationTable.ContainsKey($name) -or $existingRegistrationTable.$name.Component.Version() -ne $publicVersion) { + $registrationChanged = $true + } + if ($existingRegistrationTable.ContainsKey($name) -and $existingRegistrationTable.$name.DevelopmentDependency) { + $DevelopmentDependency = $true + } + + $registration = New-NugetComponent -Name $name -Version $publicVersion -DevelopmentDependency:$DevelopmentDependency + $RegistrationTable.Add($target, $registration) + } } } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 6e88df19dec..43a8d5d8dd4 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1637,7 +1637,7 @@ function Get-PackageDependencies function Test-Dependencies { foreach ($Dependency in "fpm") { - if (!(precheck $Dependency "Package dependency '$Dependency' not found. Run Start-PSBootstrap -Package")) { + if (!(precheck $Dependency "Package dependency '$Dependency' not found. Run Start-PSBootstrap -Scenario Package")) { # These tools are not added to the path automatically on OpenSUSE 13.2 # try adding them to the path and re-tesing first [string] $gemsPath = $null @@ -1647,7 +1647,7 @@ function Test-Dependencies $depenencyPath = Get-ChildItem -Path (Join-Path -Path $gemsPath -ChildPath "gems" -AdditionalChildPath $Dependency) -Recurse | Sort-Object -Property LastWriteTime -Descending | Select-Object -First 1 -ExpandProperty DirectoryName $originalPath = $env:PATH $env:PATH = $ENV:PATH +":" + $depenencyPath - if ((precheck $Dependency "Package dependency '$Dependency' not found. Run Start-PSBootstrap -Package")) { + if ((precheck $Dependency "Package dependency '$Dependency' not found. Run Start-PSBootstrap -Scenario Package")) { continue } else { diff --git a/tools/releaseBuild/.gitignore b/tools/releaseBuild/.gitignore deleted file mode 100644 index 0ff566888a7..00000000000 --- a/tools/releaseBuild/.gitignore +++ /dev/null @@ -1 +0,0 @@ -PSRelease/ diff --git a/tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 b/tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 deleted file mode 100644 index 2475dce7d89..00000000000 --- a/tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# PowerShell Script to build and package PowerShell from specified form and branch -# Script is intented to use in Docker containers -# Ensure PowerShell is available in the provided image - -param ( - [string] $location = "/powershell", - - # Destination location of the package on docker host - [string] $destination = '/mnt', - - [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] - [ValidateNotNullOrEmpty()] - [string]$ReleaseTag, - - [switch]$TarX64, - [switch]$TarArm, - [switch]$TarArm64, - [switch]$TarMinSize, - [switch]$FxDependent, - [switch]$Alpine -) - -$releaseTagParam = @{} -if ($ReleaseTag) -{ - $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } -} - -#Remove the initial 'v' from the ReleaseTag -$version = $ReleaseTag -replace '^v' -$semVersion = [System.Management.Automation.SemanticVersion] $version - -$metadata = Get-Content "$location/tools/metadata.json" -Raw | ConvertFrom-Json - -$LTS = $metadata.LTSRelease.Package - -Write-Verbose -Verbose -Message "LTS is set to: $LTS" - -function BuildPackages { - param( - [switch] $LTS - ) - - Push-Location - try { - Set-Location $location - Import-Module "$location/build.psm1" - Import-Module "$location/tools/packaging" - - Start-PSBootstrap -Package -NoSudo - - $buildParams = @{ Configuration = 'Release'; PSModuleRestore = $true; Restore = $true } - - if ($FxDependent.IsPresent) { - $projectAssetsZipName = 'linuxFxDependantProjectAssetssymbols.zip' - $buildParams.Add("Runtime", "fxdependent") - } elseif ($Alpine.IsPresent) { - $projectAssetsZipName = 'linuxAlpineProjectAssetssymbols.zip' - $buildParams.Add("Runtime", 'musl-x64') - } else { - # make the artifact name unique - $projectAssetsZipName = "linuxProjectAssets-$((Get-Date).Ticks)-symbols.zip" - } - - Start-PSBuild @buildParams @releaseTagParam - $options = Get-PSOptions - - if ($FxDependent) { - Start-PSPackage -Type 'fxdependent' @releaseTagParam -LTS:$LTS - } elseif ($Alpine) { - Start-PSPackage -Type 'tar-alpine' @releaseTagParam -LTS:$LTS - } else { - Start-PSPackage @releaseTagParam -LTS:$LTS - } - - if ($TarX64) { Start-PSPackage -Type tar @releaseTagParam -LTS:$LTS } - - if ($TarMinSize) { - Write-Verbose -Verbose "---- Min-Size ----" - Write-Verbose -Verbose "options.Output: $($options.Output)" - Write-Verbose -Verbose "options.Top $($options.Top)" - - $binDir = Join-Path -Path $options.Top -ChildPath 'bin' - Write-Verbose -Verbose "Remove $binDir, to get a clean build for min-size package" - Remove-Item -Path $binDir -Recurse -Force - - ## Build 'min-size' and create 'tar.gz' package for it. - $buildParams['ForMinimalSize'] = $true - Start-PSBuild @buildParams @releaseTagParam - Start-PSPackage -Type min-size @releaseTagParam -LTS:$LTS - } - - if ($TarArm) { - ## Build 'linux-arm' and create 'tar.gz' package for it. - ## Note that 'linux-arm' can only be built on Ubuntu environment. - Start-PSBuild -Configuration Release -Restore -Runtime linux-arm -PSModuleRestore @releaseTagParam - Start-PSPackage -Type tar-arm @releaseTagParam -LTS:$LTS - } - - if ($TarArm64) { - Start-PSBuild -Configuration Release -Restore -Runtime linux-arm64 -PSModuleRestore @releaseTagParam - Start-PSPackage -Type tar-arm64 @releaseTagParam -LTS:$LTS - } - } finally { - Pop-Location - } -} - -BuildPackages - -if ($LTS) { - Write-Verbose -Verbose "Packaging LTS" - BuildPackages -LTS -} - -$linuxPackages = Get-ChildItem "$location/powershell*" -Include *.deb,*.rpm,*.tar.gz - -foreach ($linuxPackage in $linuxPackages) -{ - $filePath = $linuxPackage.FullName - Write-Verbose "Copying $filePath to $destination" -Verbose - Copy-Item -Path $filePath -Destination $destination -Force -} - -Write-Verbose "Exporting project.assets files ..." -Verbose - -$projectAssetsCounter = 1 -$projectAssetsFolder = Join-Path -Path $destination -ChildPath 'projectAssets' -$projectAssetsZip = Join-Path -Path $destination -ChildPath $projectAssetsZipName -Get-ChildItem $location\project.assets.json -Recurse | ForEach-Object { - $subfolder = $_.FullName.Replace($location,'') - $subfolder.Replace('project.assets.json','') - $itemDestination = Join-Path -Path $projectAssetsFolder -ChildPath $subfolder - New-Item -Path $itemDestination -ItemType Directory -Force - $file = $_.FullName - Write-Verbose "Copying $file to $itemDestination" -Verbose - Copy-Item -Path $file -Destination "$itemDestination\" -Force - $projectAssetsCounter++ -} - -Compress-Archive -Path $projectAssetsFolder -DestinationPath $projectAssetsZip -Remove-Item -Path $projectAssetsFolder -Recurse -Force -ErrorAction SilentlyContinue diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 deleted file mode 100644 index 41ec53fa495..00000000000 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -[cmdletbinding(DefaultParameterSetName='default')] -# PowerShell Script to clone, build and package PowerShell from specified fork and branch -param ( - [string] $fork = 'powershell', - - [string] $branch = 'master', - - [string] $location = "$PWD\powershell", - - [string] $destination = "$env:WORKSPACE", - - [ValidateSet("win7-x64", "win7-x86", "win-arm", "win-arm64", "fxdependent", "fxdependent-win-desktop")] - [string] $Runtime = 'win7-x64', - - [switch] $ForMinimalSize, - - [switch] $Wait, - - [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] - [ValidateNotNullOrEmpty()] - [string] $ReleaseTag, - - [Parameter(Mandatory,ParameterSetName='IncludeSymbols')] - [switch] $Symbols, - - [Parameter(Mandatory,ParameterSetName='packageSigned')] - [ValidatePattern("-signed.zip$")] - [string] $BuildZip, - - [Parameter(Mandatory,ParameterSetName='ComponentRegistration')] - [switch] $ComponentRegistration -) - -$releaseTagParam = @{} -if ($ReleaseTag) -{ - $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } -} - -if (-not $env:homedrive) -{ - Write-Verbose "fixing empty home paths..." -Verbose - $profileParts = $env:userprofile -split ':' - $env:homedrive = $profileParts[0]+':' - $env:homepath = $profileParts[1] -} - -if (! (Test-Path $destination)) -{ - Write-Verbose "Creating destination $destination" -Verbose - $null = New-Item -Path $destination -ItemType Directory -} - -Write-Verbose "homedrive : ${env:homedrive}" -Write-Verbose "homepath : ${env:homepath}" - -# Don't use CIM_PhysicalMemory, docker containers may cache old values -$memoryMB = (Get-CimInstance win32_computersystem).TotalPhysicalMemory /1MB -$requiredMemoryMB = 2048 -if ($memoryMB -lt $requiredMemoryMB) -{ - throw "Building powershell requires at least $requiredMemoryMB MiB of memory and only $memoryMB MiB is present." -} -Write-Verbose "Running with $memoryMB MB memory." -Verbose - -try -{ - Set-Location $location - - Import-Module "$location\build.psm1" -Force - Import-Module "$location\tools\packaging" -Force - $env:platform = $null - - Write-Verbose "Sync'ing Tags..." -Verbose - Sync-PSTags -AddRemoteIfMissing - - Write-Verbose "Bootstrapping powershell build..." -Verbose - Start-PSBootstrap -Force -Package -ErrorAction Stop - - if ($PSCmdlet.ParameterSetName -eq 'packageSigned') - { - Write-Verbose "Expanding signed build..." -Verbose - if($Runtime -like 'fxdependent*') - { - Expand-PSSignedBuild -BuildZip $BuildZip -SkipPwshExeCheck - } - else - { - Expand-PSSignedBuild -BuildZip $BuildZip - } - - Remove-Item -Path $BuildZip - } - else - { - Write-Verbose "Starting powershell build for RID: $Runtime and ReleaseTag: $ReleaseTag ..." -Verbose - $buildParams = @{ - ForMinimalSize = $ForMinimalSize - } - - if($Symbols) - { - $buildParams['NoPSModuleRestore'] = $true - } - else - { - $buildParams['PSModuleRestore'] = $true - } - - Start-PSBuild -Clean -Runtime $Runtime -Configuration Release @releaseTagParam @buildParams - } - - if ($ComponentRegistration) - { - Write-Verbose "Exporting project.assets files ..." -Verbose - - $projectAssetsCounter = 1 - $projectAssetsFolder = Join-Path -Path $destination -ChildPath 'projectAssets' - $projectAssetsZip = Join-Path -Path $destination -ChildPath 'windowsProjectAssetssymbols.zip' - Get-ChildItem $location\project.assets.json -Recurse | ForEach-Object { - $subfolder = $_.FullName.Replace($location,'') - $subfolder.Replace('project.assets.json','') - $itemDestination = Join-Path -Path $projectAssetsFolder -ChildPath $subfolder - New-Item -Path $itemDestination -ItemType Directory -Force > $null - $file = $_.FullName - Write-Verbose "Copying $file to $itemDestination" -Verbose - Copy-Item -Path $file -Destination "$itemDestination\" -Force - $projectAssetsCounter++ - } - - Compress-Archive -Path $projectAssetsFolder -DestinationPath $projectAssetsZip - Remove-Item -Path $projectAssetsFolder -Recurse -Force -ErrorAction SilentlyContinue - - return - } - - if ($Runtime -like 'fxdependent*') - { - $pspackageParams = @{'Type' = $Runtime} - } - else - { - ## Set the default package type. - $pspackageParams = @{'Type' = 'msi'; 'WindowsRuntime' = $Runtime} - if ($ForMinimalSize) - { - ## Special case for the minimal size self-contained package. - $pspackageParams['Type'] = 'min-size' - } - } - - if (!$Symbols -and $Runtime -notlike 'fxdependent*' -and !$ForMinimalSize) - { - Write-Verbose "Starting powershell packaging(msi)..." -Verbose - Start-PSPackage @pspackageParams @releaseTagParam - - $pspackageParams['Type']='msix' - Write-Verbose "Starting powershell packaging(msix)..." -Verbose - Start-PSPackage @pspackageParams @releaseTagParam - } - - if ($Runtime -like 'fxdependent*' -or $ForMinimalSize) - { - ## Add symbols for just like zip package. - $pspackageParams['IncludeSymbols']=$Symbols - Start-PSPackage @pspackageParams @releaseTagParam - - ## Copy the fxdependent Zip package to destination. - Get-ChildItem $location\PowerShell-*.zip | ForEach-Object { - $file = $_.FullName - Write-Verbose "Copying $file to $destination" -Verbose - Copy-Item -Path $file -Destination "$destination\" -Force - } - } - else - { - if (!$Symbols) { - $pspackageParams['Type'] = 'zip-pdb' - Write-Verbose "Starting powershell symbols packaging(zip)..." -Verbose - Start-PSPackage @pspackageParams @releaseTagParam - } - - $pspackageParams['Type']='zip' - $pspackageParams['IncludeSymbols']=$Symbols - Write-Verbose "Starting powershell packaging(zip)..." -Verbose - Start-PSPackage @pspackageParams @releaseTagParam - - Write-Verbose "Exporting packages ..." -Verbose - - Get-ChildItem $location\*.msi,$location\*.zip,$location\*.wixpdb,$location\*.msix,$location\*.exe | ForEach-Object { - $file = $_.FullName - Write-Verbose "Copying $file to $destination" -Verbose - Copy-Item -Path $file -Destination "$destination\" -Force - } - } -} -finally -{ - Write-Verbose "Beginning build clean-up..." -Verbose - if ($Wait) - { - $path = Join-Path $PSScriptRoot -ChildPath 'delete-to-continue.txt' - $null = New-Item -Path $path -ItemType File - Write-Verbose "Computer name: $env:COMPUTERNAME" -Verbose - Write-Verbose "Delete $path to exit." -Verbose - while(Test-Path -LiteralPath $path) - { - Start-Sleep -Seconds 60 - } - } -} diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/dockerInstall.psm1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/dockerInstall.psm1 deleted file mode 100644 index 311fed7e169..00000000000 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/dockerInstall.psm1 +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -function Install-ChocolateyPackage -{ - param( - [Parameter(Mandatory=$true)] - [string] - $PackageName, - - [Parameter(Mandatory=$false)] - [string] - $Executable, - - [string[]] - $ArgumentList, - - [switch] - $Cleanup, - - [int] - $ExecutionTimeout = 2700, - - [string] - $Version - ) - - if(-not(Get-Command -Name Choco -ErrorAction SilentlyContinue)) - { - Write-Verbose "Installing Chocolatey provider..." -Verbose - Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicParsing | Invoke-Expression - } - - Write-Verbose "Installing $PackageName..." -Verbose - $extraCommand = @() - if($Version) - { - $extraCommand += '--version', $version - } - choco install -y $PackageName --no-progress --execution-timeout=$ExecutionTimeout $ArgumentList $extraCommands - - if($executable) - { - Write-Verbose "Verifing $Executable is in path..." -Verbose - $exeSource = $null - $exeSource = Get-ChildItem -Path "$env:ProgramFiles\$Executable" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty FullName - if(!$exeSource) - { - Write-Verbose "Falling back to x86 program files..." -Verbose - $exeSource = Get-ChildItem -Path "${env:ProgramFiles(x86)}\$Executable" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty FullName - } - - # Don't search the chocolatey program data until more official locations have been searched - if(!$exeSource) - { - Write-Verbose "Falling back to chocolatey..." -Verbose - $exeSource = Get-ChildItem -Path "$env:ProgramData\chocolatey\$Executable" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty FullName - } - - # all obvious locations are exhausted, use brute force and search from the root of the filesystem - if(!$exeSource) - { - Write-Verbose "Falling back to the root of the drive..." -Verbose - $exeSource = Get-ChildItem -Path "/$Executable" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty FullName - } - - if(!$exeSource) - { - throw "$Executable not found" - } - - $exePath = Split-Path -Path $exeSource - Append-Path -path $exePath - } - - if($Cleanup.IsPresent) - { - Remove-Folder -Folder "$env:temp\chocolatey" - } -} - -function Append-Path -{ - param - ( - $path - ) - $machinePathString = [System.Environment]::GetEnvironmentVariable('path',[System.EnvironmentVariableTarget]::Machine) - $machinePath = $machinePathString -split ';' - - if($machinePath -inotcontains $path) - { - $newPath = "$machinePathString;$path" - Write-Verbose "Adding $path to path..." -Verbose - [System.Environment]::SetEnvironmentVariable('path',$newPath,[System.EnvironmentVariableTarget]::Machine) - Write-Verbose "Added $path to path." -Verbose - } - else - { - Write-Verbose "$path already in path." -Verbose - } -} - -function Remove-Folder -{ - param( - [string] - $Folder - ) - - Write-Verbose "Cleaning up $Folder..." -Verbose - $filter = Join-Path -Path $Folder -ChildPath * - [int]$measuredCleanupMB = (Get-ChildItem $filter -Recurse | Measure-Object -Property Length -Sum).Sum / 1MB - Remove-Item -Recurse -Force $filter -ErrorAction SilentlyContinue - Write-Verbose "Cleaned up $measuredCleanupMB MB from $Folder" -Verbose -} diff --git a/tools/releaseBuild/README.md b/tools/releaseBuild/README.md deleted file mode 100644 index 9b78e742b5f..00000000000 --- a/tools/releaseBuild/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Azure Dev Ops Release Builds - -## Requirements - -Docker must be installed to run any of the release builds. - -## Running Windows Release Builds Locally - -From PowerShell on Windows, run `.\vstsbuild.ps1 -ReleaseTag -Name `. - -For the package builds, run `.\vstsbuild.ps1 -ReleaseTag -Name -BuildPath -SignedFilesPath ` - -Windows Build Names: - -* `win7-x64-symbols` - * Builds the Windows x64 Zip with symbols -* `win7-x86-symbols` - * Builds the Windows x86 Zip with symbols -* `win7-arm-symbols` - * Builds the Windows ARM Zip with symbols -* `win7-arm64-symbols` - * Builds the Windows ARM64 Zip with symbols -* `win7-fxdependent-symbols` - * Builds the Windows FxDependent Zip with symbols -* `win7-x64-package` - * Builds the Windows x64 packages -* `win7-x86-package` - * Builds the Windows x86 packages -* `win7-arm-package` - * Builds the Windows ARM packages -* `win7-arm64-package` - * Builds the Windows ARM64 packages -* `win7-fxdependent-package` - * Builds the Windows FxDependent packages - -## Running Linux Release Builds Locally - -From PowerShell on Linux or macOS, run `.\vstsbuild.ps1 -ReleaseTag -Name `. - -Linux Build Names: - -* `deb` - * Builds the Debian Packages, ARM32 and ARM64. -* `alpine` - * Builds the Alpine Package -* `rpm` - * Builds the RedHat variant Package - -## Azure Dev Ops Build - -The release build is fairly complicated. The definition is at `./azureDevOps/releaseBuild.yml`. - -Here is a diagram of the build: - -[![Release Build diagram](https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/releaseBuild/azureDevOps/diagram.svg?sanitize=true)](https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/releaseBuild/azureDevOps/diagram.svg?sanitize=true) diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml deleted file mode 100644 index da26ea6d348..00000000000 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Sync packages from PSGallery to Azure DevOps Artifacts feed - -resources: -- repo: self - clean: true - -pool: - name: 1es - demands: - - ImageOverride -equals PSMMS2019-Minimal - -steps: - - pwsh: | - $minVer = [version]"2.2.3" - $curVer = Get-Module PowerShellGet -ListAvailable | Select-Object -First 1 | ForEach-Object Version - if (-not $curVer -or $curVer -lt $minVer) { - Install-Module -Name PowerShellGet -MinimumVersion 2.2.3 -Force - } - displayName: Update PSGet and PackageManagement - condition: succeededOrFailed() - - - pwsh: | - Write-Verbose -Verbose "Packages to upload" - if(Test-Path $(Build.ArtifactStagingDirectory)) { Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg" | ForEach-Object { $_.FullName }} - displayName: List packages to upload - condition: succeededOrFailed() - - - task: NuGetCommand@2 - displayName: 'NuGet push' - inputs: - command: push - publishVstsFeed: 'pscore-release' - publishFeedCredentials: 'AzArtifactsFeed' diff --git a/tools/releaseBuild/azureDevOps/compliance.yml b/tools/releaseBuild/azureDevOps/compliance.yml deleted file mode 100644 index 3624f1e1081..00000000000 --- a/tools/releaseBuild/azureDevOps/compliance.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Compliance-$(Build.BuildId) - -trigger: none -pr: none - -schedules: - # Chrontab format, see https://en.wikipedia.org/wiki/Cron - # this is in UTC - - cron: '0 13 * * *' - branches: - include: - - master - -resources: - repositories: - - repository: ComplianceRepo - type: github - endpoint: ComplianceGHRepo - name: PowerShell/compliance - ref: master - -parameters: -- name: InternalSDKBlobURL - displayName: URL to the blob havibg internal .NET SDK - type: string - default: ' ' - -variables: - - name: DOTNET_CLI_TELEMETRY_OPTOUT - value: 1 - - name: POWERSHELL_TELEMETRY_OPTOUT - value: 1 - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - # Defines the variables AzureFileCopySubscription, StorageAccount, StorageAccountKey, StorageResourceGroup, StorageSubscriptionName - - group: 'Azure Blob variable group' - # Defines the variables CgPat, CgOrganization, and CgProject - - group: 'ComponentGovernance' - - group: 'PoolNames' - - name: __DOTNET_RUNTIME_FEED - value: ${{ parameters.InternalSDKBlobURL }} - - -stages: - - stage: compliance - displayName: 'Compliance' - dependsOn: [] - jobs: - - template: templates/compliance/compliance.yml - parameters: - parentJobs: [] - - stage: APIScan - displayName: 'ApiScan' - dependsOn: [] - jobs: - - template: templates/compliance/apiscan.yml - parameters: - parentJobs: [] - - stage: notice - displayName: Generate Notice File - dependsOn: [] - jobs: - - template: templates/compliance/generateNotice.yml - parameters: - parentJobs: [] diff --git a/tools/releaseBuild/azureDevOps/diagram.puml b/tools/releaseBuild/azureDevOps/diagram.puml deleted file mode 100644 index ade53b11b9c..00000000000 --- a/tools/releaseBuild/azureDevOps/diagram.puml +++ /dev/null @@ -1,107 +0,0 @@ -@startuml - -folder "Linux Builds" as LinuxBuilds { - ' Define the build tasks as business processes - agent "DEB" as BuildDEB - agent "RPM" as BuildRPM - agent "Alpine" as BuildAlpine - agent "Linux-FxDependent" as BuildLinuxFx - -} - -agent "macOS Build" as BuildMac - -agent "Upload build metadata" as BuildMetadata - -folder "Windows Builds" as WinBuilds { - agent "x64" as BuildWinX64 - agent "x86" as BuildWinX86 - agent "arm32" as BuildWinArm32 - agent "arm64" as BuildWinArm64 - agent "FxDependent" as BuildWinFx -} - -agent "ComponentRegistration" as BuildCG - -folder "Linux Package Scanning and Upload" as PkgScanUploadLinux { - agent "DEB" as UploadDEB - agent "RPM" as UploadRPM - agent "Alpine" as UploadAlpine - agent "Linux-FxDependent" as UploadLinuxFx -} - -folder "Package Signing and Upload" as PkgSignUpload { - agent "macOS" as SignMac - - agent "Windows" as SignWin -} - -folder "Build Test Artifacts" as TestArtifacts { - agent "Windows" as WinTest - agent "Linux" as LinuxTest - agent "Linux-ARM" as LinuxArmTest - agent "Linux-ARM64" as LinuxArm64Test -} - -agent "Compliance" as Compliance - - -agent "Create SDK and Global Tool and Upload" as BuildNuGet - - -' Define finishing the build as a goal filled -control "Finish" as Finish -control "Start" as Start - -' map the various Upload task dependencies -BuildDEB -down-> UploadDEB -BuildRPM -down-> UploadRPM -BuildLinuxFx -down-> UploadLinuxFx -BuildAlpine -down-> UploadAlpine - -' map all of the SignMac task dependencies -BuildMac -down-> SignMac - -' map all of the SignWin task dependencies -WinBuilds -down-> SignWin -'BuildWinX64 -down-> SignWin -'BuildWinX86 -down-> SignWin -'BuildWinArm32 -down-> SignWin -'BuildWinArm64 -down-> SignWin -'BuildWinFx -down-> SignWin - -' map all of the Compliance task dependencies -BuildWinX86 -down-> Compliance -BuildWinX64 -down-> Compliance -BuildWinFx -down-> Compliance - -PkgSignUpload -down-> BuildNuGet -LinuxBuilds -down-> BuildNuGet - -' map all leafs to finish -Compliance ~~ Finish -UploadAlpine ~~ Finish -UploadDEB ~~ Finish -UploadRPM ~~ Finish -UploadLinuxFx ~~ Finish -SignMac ~~ Finish -BuildCG ~~ Finish -BuildNuGet ~~ Finish -TestArtifacts ~~ Finish -BuildMetadata ~~ Finish - -Start ~~ BuildDEB -Start ~~ BuildRPM -Start ~~ BuildAlpine -Start ~~ BuildLinuxFx -Start ~~ BuildMac -Start ~~ BuildWinX64 -Start ~~ BuildWinX86 -Start ~~ BuildWinFx -Start ~~ BuildWinArm32 -Start ~~ BuildWinArm64 -Start ~~ BuildCG -Start ~~ TestArtifacts -Start ~~ BuildMetadata - -@enduml diff --git a/tools/releaseBuild/azureDevOps/diagram.svg b/tools/releaseBuild/azureDevOps/diagram.svg deleted file mode 100644 index 024128bf988..00000000000 --- a/tools/releaseBuild/azureDevOps/diagram.svg +++ /dev/null @@ -1,108 +0,0 @@ -Linux BuildsWindows BuildsLinux Package Scanning and UploadPackage Signing and UploadBuild Test ArtifactsDEBRPMAlpineLinux-FxDependentx64x86arm32arm64FxDependentDEBRPMAlpineLinux-FxDependentmacOSWindowsWindowsLinuxLinux-ARMLinux-ARM64macOS BuildUpload build metadataComponentRegistrationComplianceCreate SDK and Global Tool and UploadFinishStart \ No newline at end of file diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml deleted file mode 100644 index 3be90bbefbc..00000000000 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ /dev/null @@ -1,379 +0,0 @@ -name: UnifiedPackageBuild-$(Build.BuildId) -trigger: - branches: - include: - - master - - release* -pr: - branches: - include: - - master - - release* - -parameters: - - name: ForceAzureBlobDelete - displayName: Delete Azure Blob - type: string - values: - - true - - false - default: false - - name: InternalSDKBlobURL - displayName: URL to the blob having internal .NET SDK - type: string - default: ' ' - -resources: - repositories: - - repository: ComplianceRepo - type: github - endpoint: ComplianceGHRepo - name: PowerShell/compliance - ref: master - -variables: - - name: PS_RELEASE_BUILD - value: 1 - - name: DOTNET_CLI_TELEMETRY_OPTOUT - value: 1 - - name: POWERSHELL_TELEMETRY_OPTOUT - value: 1 - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - # Prevents auto-injection of nuget-security-analysis@0 - - name: skipNugetSecurityAnalysis - value: true - - name: branchCounterKey - value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - - name: branchCounter - value: $[counter(variables['branchCounterKey'], 1)] - - name: ForceAzureBlobDelete - value: ${{ parameters.ForceAzureBlobDelete }} - - name: Github_Build_Repository_Uri - value: https://github.com/powershell/powershell - - name: SBOMGenerator_Formats - value: spdx:2.2 - - name: BUILDSECMON_OPT_IN - value: true - - group: PoolNames - - name: __DOTNET_RUNTIME_FEED - value: ${{ parameters.InternalSDKBlobURL }} - -stages: - - stage: prep - jobs: - - template: templates/checkAzureContainer.yml - - - stage: macos - dependsOn: ['prep'] - jobs: - - template: templates/mac.yml - parameters: - buildArchitecture: x64 - - - template: templates/mac.yml - parameters: - buildArchitecture: arm64 - - - stage: linux - dependsOn: ['prep'] - jobs: - - template: templates/linux.yml - parameters: - buildName: deb - - - template: templates/linux.yml - parameters: - buildName: rpm - parentJob: build_deb - - - template: templates/linux.yml - parameters: - buildName: fxdependent - parentJob: build_deb - - - template: templates/linux.yml - parameters: - buildName: alpine - - - stage: windows - dependsOn: ['prep'] - jobs: - - template: templates/windows-hosted-build.yml - parameters: - Architecture: x64 - - - template: templates/windows-hosted-build.yml - parameters: - Architecture: x64 - BuildConfiguration: minSize - - - template: templates/windows-hosted-build.yml - parameters: - Architecture: x86 - - - template: templates/windows-hosted-build.yml - parameters: - Architecture: arm64 - - - template: templates/windows-hosted-build.yml - parameters: - Architecture: fxdependent - - - template: templates/windows-hosted-build.yml - parameters: - Architecture: fxdependentWinDesktop - - - stage: SignFiles - displayName: Sign files - dependsOn: ['windows', 'linux', 'macos'] - jobs: - - template: templates/mac-file-signing.yml - parameters: - buildArchitecture: x64 - - - template: templates/mac-file-signing.yml - parameters: - buildArchitecture: arm64 - - - job: SignFilesWinLinux - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - displayName: Sign files - - variables: - - group: ESRP - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: repoFolder - value: PowerShell - - name: repoRoot - value: $(Agent.BuildDirectory)\$(repoFolder) - - name: complianceRepoFolder - value: compliance - - strategy: - matrix: - linux-x64: - runtime: linux-x64 - unsignedBuildArtifactContainer: pwshLinuxBuild.tar.gz - unsignedBuildArtifactName: pwshLinuxBuild.tar.gz - signedBuildArtifactName: pwshLinuxBuild.tar.gz - signedArtifactContainer: authenticode-signed - linux-x64-Alpine: - runtime: linux-x64-Alpine - unsignedBuildArtifactContainer: pwshLinuxBuildAlpine.tar.gz - unsignedBuildArtifactName: pwshLinuxBuild.tar.gz - signedBuildArtifactName: pwshLinuxBuildAlpine.tar.gz - signedArtifactContainer: authenticode-signed - linux-x64-Alpine-Fxdependent: - runtime: linux-x64-Alpine-Fxdependent - unsignedBuildArtifactContainer: pwshAlpineFxdBuildAmd64.tar.gz - unsignedBuildArtifactName: pwshAlpineFxdBuildAmd64.tar.gz - signedBuildArtifactName: pwshAlpineFxdBuildAmd64.tar.gz - signedArtifactContainer: authenticode-signed - linux-arm32: - runtime: linux-arm32 - unsignedBuildArtifactContainer: pwshLinuxBuildArm32.tar.gz - unsignedBuildArtifactName: pwshLinuxBuildArm32.tar.gz - signedBuildArtifactName: pwshLinuxBuildArm32.tar.gz - signedArtifactContainer: authenticode-signed - linux-arm64: - runtime: linux-arm64 - unsignedBuildArtifactContainer: pwshLinuxBuildArm64.tar.gz - unsignedBuildArtifactName: pwshLinuxBuildArm64.tar.gz - signedBuildArtifactName: pwshLinuxBuildArm64.tar.gz - signedArtifactContainer: authenticode-signed - linux-fxd: - runtime: linux-fxd - unsignedBuildArtifactContainer: pwshLinuxBuildFxdependent.tar.gz - unsignedBuildArtifactName: pwshLinuxBuild.tar.gz - signedBuildArtifactName: pwshLinuxBuildFxdependent.tar.gz - signedArtifactContainer: authenticode-signed - linux-mariner: - runtime: linux-mariner - unsignedBuildArtifactContainer: pwshMarinerBuildAmd64.tar.gz - unsignedBuildArtifactName: pwshMarinerBuildAmd64.tar.gz - signedBuildArtifactName: pwshMarinerBuildAmd64.tar.gz - signedArtifactContainer: authenticode-signed - linux-arm64-mariner: - runtime: linux-arm64-mariner - unsignedBuildArtifactContainer: pwshMarinerBuildArm64.tar.gz - unsignedBuildArtifactName: pwshMarinerBuildArm64.tar.gz - signedBuildArtifactName: pwshMarinerBuildArm64.tar.gz - signedArtifactContainer: authenticode-signed - linux-minsize: - runtime: linux-minsize - unsignedBuildArtifactContainer: pwshLinuxBuildMinSize.tar.gz - unsignedBuildArtifactName: pwshLinuxBuildMinSize.tar.gz - signedBuildArtifactName: pwshLinuxBuildMinSize.tar.gz - signedArtifactContainer: authenticode-signed - win-x64: - runtime: win-x64 - unsignedBuildArtifactContainer: results - unsignedBuildArtifactName: '**/*-symbols-win-x64.zip' - signedBuildArtifactName: '-symbols-win-x64-signed.zip' - signedArtifactContainer: results - win-x86: - runtime: win-x86 - unsignedBuildArtifactContainer: results - unsignedBuildArtifactName: '**/*-symbols-win-x86.zip' - signedBuildArtifactName: '-symbols-win-x86-signed.zip' - signedArtifactContainer: results - win-arm64: - runtime: win-arm64 - unsignedBuildArtifactContainer: results - unsignedBuildArtifactName: '**/*-symbols-win-arm64.zip' - signedBuildArtifactName: '-symbols-win-arm64-signed.zip' - signedArtifactContainer: results - win-x64-gc: - runtime: win-x64-gc - unsignedBuildArtifactContainer: results - unsignedBuildArtifactName: '**/*-symbols-win-x64-gc.zip' - signedBuildArtifactName: '-symbols-win-x64-gc-signed.zip' - signedArtifactContainer: results - win-fxdependent: - runtime: win-fxdependent - unsignedBuildArtifactContainer: results - unsignedBuildArtifactName: '**/*-symbols-win-fxdependent.zip' - signedBuildArtifactName: '-symbols-win-fxdependent-signed.zip' - signedArtifactContainer: results - win-fxdependentWinDesktop: - runtime: win-fxdependentWinDesktop - unsignedBuildArtifactContainer: results - unsignedBuildArtifactName: '**/*-symbols-win-fxdependentWinDesktop.zip' - signedBuildArtifactName: '-symbols-win-fxdependentWinDesktop-signed.zip' - signedArtifactContainer: results - steps: - - template: templates/sign-build-file.yml - - - stage: mac_packaging - displayName: macOS packaging - dependsOn: ['SignFiles'] - jobs: - - template: templates/mac-package-build.yml - parameters: - buildArchitecture: x64 - - - template: templates/mac-package-build.yml - parameters: - buildArchitecture: arm64 - - - stage: linux_packaging - displayName: Linux Packaging - dependsOn: ['SignFiles'] - jobs: - - template: templates/linux-packaging.yml - parameters: - buildName: deb - - - template: templates/linux-packaging.yml - parameters: - buildName: rpm - uploadDisplayName: Upload and Sign - - - template: templates/linux-packaging.yml - parameters: - buildName: alpine - - - template: templates/linux-packaging.yml - parameters: - buildName: fxdependent - - - stage: win_packaging - displayName: Windows Packaging - dependsOn: ['SignFiles'] - jobs: - - template: templates/windows-packaging.yml - parameters: - Architecture: x64 - parentJob: build_windows_x64_release - - - template: templates/windows-packaging.yml - parameters: - Architecture: x64 - BuildConfiguration: minSize - parentJob: build_windows_x64_minSize - - - template: templates/windows-packaging.yml - parameters: - Architecture: x86 - parentJob: build_windows_x86_release - - - template: templates/windows-packaging.yml - parameters: - Architecture: arm64 - parentJob: build_windows_arm64_release - - - template: templates/windows-packaging.yml - parameters: - Architecture: fxdependent - parentJob: build_windows_fxdependent_release - - - template: templates/windows-packaging.yml - parameters: - Architecture: fxdependentWinDesktop - parentJob: build_windows_fxdependentWinDesktop_release - - - stage: package_signing - displayName: Package Signing - dependsOn: ['mac_packaging', 'linux_packaging', 'win_packaging'] - jobs: - - template: templates/windows-package-signing.yml - - - template: templates/mac-package-signing.yml - parameters: - buildArchitecture: x64 - - - template: templates/mac-package-signing.yml - parameters: - buildArchitecture: arm64 - - - stage: nuget_and_json - displayName: NuGet Packaging and Build Json - dependsOn: ['package_signing'] - jobs: - - template: templates/nuget.yml - - template: templates/json.yml - - # This is done late so that we dont use resources before the big signing and packaging tasks. - - stage: compliance - dependsOn: ['package_signing'] - jobs: - - template: templates/compliance.yml - - - stage: test_and_release_artifacts - displayName: Test and Release Artifacts - dependsOn: ['prep'] - jobs: - - template: templates/testartifacts.yml - - - job: release_json - displayName: Create and Upload release.json - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - steps: - - checkout: self - clean: true - - template: templates/SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - powershell: | - $metadata = Get-Content '$(Build.SourcesDirectory)/tools/metadata.json' -Raw | ConvertFrom-Json - $LTS = $metadata.LTSRelease.Package - @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" - Get-Content "$(Build.StagingDirectory)\release.json" - Write-Host "##vso[artifact.upload containerfolder=metadata;artifactname=metadata]$(Build.StagingDirectory)\release.json" - displayName: Create and upload release.json file to build artifact - retryCountOnTaskFailure: 2 - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml deleted file mode 100644 index e21f6d590fe..00000000000 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ /dev/null @@ -1,673 +0,0 @@ -trigger: none - -# needed to disable CI trigger and allow manual trigger -# when the branch is same as pipeline source, the latest build from the source is used. -# all environment used are for manual tasks and approvals. - -parameters: - - name: skipPackagesMsftComPublish - displayName: Skip actual publishing to Packages.microsoft.com, AFTER we upload it. Used to test the publishing script. - default: false - type: boolean - - name: skipNugetPublish - displayName: Skip nuget publishing. Used in testing publishing stage. - default: false - type: boolean - -resources: - pipelines: - - pipeline: releasePipeline - source: 'Coordinated Packages' - trigger: - branches: - - release/* - - repositories: - - repository: Internal-PowerShellTeam-Tools - type: git - trigger: none - name: Internal-PowerShellTeam-Tools - ref: main-mirror - - - repository: ComplianceRepo - type: github - endpoint: ComplianceGHRepo - name: PowerShell/compliance - ref: master - -variables: - - name: runCodesignValidationInjection - value : false - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: skipComponentGovernanceDetection - value: true - - name: BUILDSECMON_OPT_IN - value: true - - group: ReleasePipelineSecrets - - group: PipelineExecutionPats - -stages: -- stage: MSIXBundle - displayName: Create MSIX Bundle package - dependsOn: [] - jobs: - - template: templates/release-MsixBundle.yml - -- stage: ValidateSDK - displayName: Validate SDK - dependsOn: [] - jobs: - - template: templates/release-SDKTests.yml - parameters: - jobName: WinSDK - displayName: Windows SDK Test - imageName: windows-latest - - - template: templates/release-SDKTests.yml - parameters: - jobName: LinuxSDK - displayName: Linux SDK Test - imageName: ubuntu-latest - - - template: templates/release-SDKTests.yml - parameters: - jobName: macOSSDK - displayName: macOS SDK Test - imageName: macOS-latest - -- stage: PRCreation - displayName: Create PR in GH Master - dependsOn: [] - jobs: - - deployment: CreatePRInMaster - displayName: Update README.md and metadata.json - pool: server - environment: PSReleaseCreatePR - -- stage: ValidateGlobalTool - displayName: Validate Global Tool - dependsOn: [] - jobs: - - template: templates/release-GlobalToolTest.yml - parameters: - jobName: WinGblTool - displayName: Global Tool Test Windows - imageName: windows-latest - globalToolExeName: 'pwsh.exe' - globalToolPackageName: 'PowerShell.Windows.x64' - - - template: templates/release-GlobalToolTest.yml - parameters: - jobName: LinuxWinGblTool - displayName: Global Tool Test Linux - imageName: ubuntu-latest - globalToolExeName: 'pwsh' - globalToolPackageName: 'PowerShell.Linux.x64' - -- stage: ValidateFxdPackage - displayName: Validate Fxd Package - dependsOn: [] - jobs: - - template: templates/release-ValidateFxdPackage.yml - parameters: - jobName: WinFxdPackage - displayName: Fxd Package Test Win - imageName: windows-latest - packageNamePattern: '**/*win-fxdependent.zip' - - - template: templates/release-ValidateFxdPackage.yml - parameters: - jobName: FxdPackageWindDesktop - displayName: Fxd Package Test WinDesktop - imageName: windows-latest - packageNamePattern: '**/*win-fxdependentWinDesktop.zip' - - - template: templates/release-ValidateFxdPackage.yml - parameters: - jobName: FxdPackageLinux - displayName: Fxd Package Test Linux - imageName: ubuntu-latest - packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' - - - template: templates/release-ValidateFxdPackage.yml - parameters: - jobName: FxdPackageLinuxonARM - displayName: Fxd Package Test Linux ARM64 - imageName: 'PSMMSUbuntu20.04-ARM64-secure' - packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' - use1ES: true - -- stage: StaticPkgValidation - dependsOn: [] - displayName: Static package validation - jobs: - - job: ValidatePkgNames - displayName: Validate Package Names - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - variables: - - group: 'Azure Blob variable group' - steps: - - template: templates/release-ValidatePackageNames.yml - - job: ValidatePkgBOM - displayName: Validate Package BOM - pool: - # testing - vmImage: ubuntu-latest - steps: - - template: templates/release-ValidatePackageBOM.yml - -- stage: StartDocker - dependsOn: [] - displayName: Kick Off Docker Staging build - jobs: - - deployment: PSDockerKickOff - displayName: Start Docker build - pool: server - environment: PSReleaseDockerKickOff - -- stage: ManualValidation - dependsOn: [] - displayName: Manual Validation - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Validate Windows Packages - jobName: ValidateWinPkg - instructions: | - Validate zip and msipackages on Windows Server 2012 R2 - - - template: templates/release/approvalJob.yml - parameters: - displayName: Validate OSX Packages - jobName: ValidateOsxPkg - instructions: | - Validate tar.gz package on osx-arm64 - -- stage: ReleaseAutomation - displayName: Release Automation - dependsOn: [] - jobs: - - job: KickOffRA - displayName: Kickoff Release Automation - timeoutInMinutes: 240 - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: Internal-PowerShellTeam-Tools - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - - - pwsh: | - Get-ChildItem -Path $(Build.SourcesDirectory) - Import-Module $(Build.SourcesDirectory)\ReleaseTools\AzDO -Force - Set-AzDoProjectInfo -ProjectOwner PowerShell-Rel -ProjectName Release-Automation - Set-AzDoAuthToken -Token $(powershellRelExecutionPat) - $packageBuildID = $(resources.pipeline.releasePipeline.runID) - $metadata = Get-Content -Raw -Path '$(Pipeline.Workspace)/releasePipeline/metadata/release.json' | ConvertFrom-Json - $buildInvocationInfo = Start-AzDOBuild -BuildDefinitionId 10 -BuildArguments @{ POWERSHELL_PACKAGE_BUILD_BUILDID = $packageBuildID } -Tag $metadata.ReleaseVersion, 'InProgress' -PassThru - Write-Verbose -Verbose "Kicked off release automation:`n$($buildInvocationInfo | Out-String)" - $status = $buildInvocationInfo | Wait-AzDOBuildStatus -Status Completed -timeoutMinutes 240 - if ($status.result -ne 'Succeeded') { - Write-Verbose "There are errors in release automation tests. Please triage failures." - } - - - template: templates/release/approvalJob.yml - parameters: - displayName: Triage Release Automation Results - jobName: TriageRA - dependsOnJob: KickOffRA - instructions: | - Validate all the test failures and continue when signed off - - - job: MarkRASignOff - displayName: Mark release automation signoff - dependsOn: TriageRA - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: Internal-PowerShellTeam-Tools - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - - - pwsh: | - Import-Module $(Build.SourcesDirectory)\ReleaseTools\AzDO -Force - Set-AzDoProjectInfo -ProjectOwner PowerShell-Rel -ProjectName Release-Automation - Set-AzDoAuthToken -Token $(powershellRelExecutionPat) - $metadata = Get-Content -Raw -Path '$(Pipeline.Workspace)/releasePipeline/metadata/release.json' | ConvertFrom-Json - $azDOBuild = Get-AzDOBuild -buildDefinitionId 10 -MaximumResult 100 | Where-Object { $_.tags -in $metadata.ReleaseVersion } - $azDoBuild | Remove-AzDOBuildTag -tag 'InProgress' -Pass | Add-AzDOBuildTag -tag 'SignedOff' - displayName: Signoff Release-Automation run - -- stage: UpdateChangeLog - displayName: Update the changelog - # do not include stages that are likely to fail in dependency as there is no way to force deploy. - dependsOn: - - MSIXBundle - - ValidateSDK - - PRCreation - - StaticPkgValidation - - StartDocker - - ManualValidation - - ValidateFxdPackage - - ValidateGlobalTool - - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Make sure the changelog is updated - jobName: MergeChangeLog - instructions: | - Update and merge the changelog for the release. - This step is required for creating GitHub draft release. - -- stage: BlobPublic - displayName: Make Blob Public - # do not include stages that are likely to fail in dependency as there is no way to force deploy. - dependsOn: UpdateChangeLog - - # The environment here is used for approval. - jobs: - - deployment: AzureBlobPublic - displayName: Make Azure Blob Public - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: 'Staging_ACR' - environment: PSReleaseAzureBlobPublic - strategy: - runOnce: - deploy: - steps: - - template: templates/release-MakeContainerPublic.yml - - - template: templates/release/approvalJob.yml - parameters: - displayName: Copy Global tool packages to PSInfra storage - jobName: CopyBlobApproval - instructions: | - Approval for Copy global tool packages to PSInfra storage - - - job: PSInfraBlobPublic - displayName: Copy global tools to PSInfra storage - dependsOn: CopyBlobApproval - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: 'PSInfraStorage' - - steps: - - template: templates/release-CopyGlobalTools.yml - parameters: - sourceContainerName: 'tool-private' - destinationContainerName: 'tool' - sourceStorageAccountName: '$(GlobalToolStorageAccount)' - destinationStorageAccountName: '$(PSInfraStorageAccount)' - blobPrefix: '$(Version)' - -- stage: GitHubTasks - displayName: GitHub tasks - dependsOn: BlobPublic - jobs: - - job: GitHubDraft - displayName: Create GitHub Draft release - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: 'Azure Blob variable group' - - group: mscodehub-feed-read-general - - group: mscodehub-feed-read-akv - - group: ReleasePipelineSecrets - steps: - - template: templates/release-CreateGitHubDraft.yml - - - deployment: PushTag - dependsOn: GitHubDraft - displayName: Push Git Tag - pool : server - environment: PSReleasePushTag - - - deployment: MakeDraftPublic - dependsOn: PushTag - displayName: Make GitHub Draft public - pool : server - environment: PSReleaseDraftPublic - -- stage: PublishPackages - displayName: Publish packages - dependsOn: GitHubTasks - jobs: - - job: PublishNuget - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - template: templates/release-ReleaseToNuGet.yml - parameters: - skipPublish: ${{ parameters.skipNugetPublish }} - - - job: PublishPkgsMsftCom - - timeoutInMinutes: 120 - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMSUbuntu20.04-Secure - - variables: - - group: mscodehub-feed-read-general - - group: mscodehub-feed-read-akv - - group: 'packages.microsoft.com' - - group: 'mscodehub-code-read-akv' - steps: - - template: templates/release-PublishPackageMsftCom.yml - parameters: - skipPublish: ${{ parameters.skipPackagesMsftComPublish }} - -- stage: PublishSymbols - displayName: Publish symbols - dependsOn: PublishPackages - jobs: - - job: PublishSymbol - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - template: templates/release-PublishSymbols.yml - -- stage: ChangesToMaster - displayName: Ensure changes are in GH master - dependsOn: PublishPackages - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Make sure changes are in master - jobName: MergeToMaster - instructions: | - Make sure that changes README.md and metadata.json are merged into master on GitHub. - -- stage: ReleaseDocker - displayName: Release Docker - dependsOn: - - GitHubTasks - jobs: - - deployment: ReleaseDocker - displayName: Release Docker - pool: server - environment: PSReleaseDockerRelease - -- stage: ReleaseSnap - displayName: Release Snap - dependsOn: - - PublishPackages - - ChangesToMaster - variables: - # adds newPwshOrgName (exists in new and old org) - - group: PowerShellRelease - jobs: - - job: KickoffSnap - displayName: Kickoff Snap build - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: Internal-PowerShellTeam-Tools - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - - pwsh: | - Import-Module $(Build.SourcesDirectory)\ReleaseTools\AzDO -Force - Set-AzDoProjectInfo -ProjectOwner PowerShell-Rel -ProjectName PowerShell - Set-AzDoAuthToken -Token $(powershellRelExecutionPat) - $metadata = Get-Content -Raw -Path '$(Pipeline.Workspace)/releasePipeline/metadata/release.json' | ConvertFrom-Json - $buildInvocationInfo = Start-AzDOBuild -BuildDefinitionId 49 -Tag $metadata.ReleaseVersion, 'InProgress' -PassThru - Write-Verbose -Verbose "Kicked off snap build: $($buildInvocationInfo.WebUrl)" - $status = $buildInvocationInfo | Wait-AzDOBuildStatus -Status Completed -timeoutMinutes 60 - if ($status.result -ne 'Succeeded') { - throw "There are errors in snap build!!" - } - - - template: templates/release/approvalJob.yml - parameters: - displayName: Approve the release - jobName: SnapEnd - dependsOnJob: KickoffSnap - instructions: | - Once the build is finished, approve the release of all channels. - - - job: MarkSnapSignOff - displayName: Mark release automation signoff - dependsOn: SnapEnd - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: Internal-PowerShellTeam-Tools - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - - pwsh: | - Import-Module $(Build.SourcesDirectory)\ReleaseTools\AzDO -Force - Set-AzDoProjectInfo -ProjectOwner PowerShell-Rel -ProjectName PowerShell - Set-AzDoAuthToken -Token $(powershellRelExecutionPat) - $metadata = Get-Content -Raw -Path '$(Pipeline.Workspace)/releasePipeline/metadata/release.json' | ConvertFrom-Json - $azDOBuild = Get-AzDOBuild -buildDefinitionId 49 -MaximumResult 100 | Where-Object { $_.tags -in $metadata.ReleaseVersion } - $azDoBuild | Remove-AzDOBuildTag -tag 'InProgress' -Pass | Add-AzDOBuildTag -tag 'SignedOff' - displayName: Signoff Release-Automation run - -- stage: ReleaseToMU - displayName: Release to MU - dependsOn: - - PublishPackages - - ChangesToMaster - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Release to MU - instructions: | - Notify the PM team to start the process of releasing to MU. - -- stage: UpdateDotnetDocker - dependsOn: GitHubTasks - displayName: Update DotNet SDK Docker images - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Update .NET SDK docker images - jobName: DotnetDocker - instructions: | - Create PR for updating dotnet-docker images to use latest PowerShell version. - 1. Fork and clone https://github.com/dotnet/dotnet-docker.git - 2. git checkout upstream/nightly -b updatePS - 3. dotnet run --project .\eng\update-dependencies\ -- --product-version powershell= --compute-shas - 4. create PR targeting nightly branch - -- stage: UpdateWinGet - dependsOn: GitHubTasks - displayName: Add manifest entry to winget - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Add manifest entry to winget - jobName: UpdateWinGet - instructions: | - This is typically done by the community 1-2 days after the release. - -- stage: PublishMsix - dependsOn: GitHubTasks - displayName: Publish MSIX to store - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Publish the MSIX Bundle package to store - jobName: PublishMsix - instructions: | - Ask Steve to release MSIX bundle package to Store - -- stage: BuildInfoJson - dependsOn: GitHubTasks - displayName: Upload BuildInfoJson - jobs: - - deployment: UploadJson - displayName: Upload BuildInfoJson - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: 'Azure Blob variable group' - environment: PSReleaseBuildInfoJson - strategy: - runOnce: - deploy: - steps: - - template: templates/release-BuildJson.yml - -- stage: ReleaseVPack - dependsOn: GitHubTasks - displayName: Release VPack - jobs: - - job: KickoffvPack - displayName: Kickoff vPack build - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: Internal-PowerShellTeam-Tools - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - - - pwsh: | - Import-Module $(Build.SourcesDirectory)\ReleaseTools\AzDO -Force - Set-AzDoProjectInfo -ProjectOwner mscodehub -ProjectName PowerShellCore - Set-AzDoAuthToken -Token $(mscodehubBuildExecutionPat) - $metadata = Get-Content -Raw -Path '$(Pipeline.Workspace)/releasePipeline/metadata/release.json' | ConvertFrom-Json - $releaseVersion = $metadata.ReleaseVersion -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - if (-not $isPreview) { - $buildInvocationInfo = Start-AzDOBuild -BuildDefinitionId 1238 -Branch '$(Build.SourceBranch)' -Tag $metadata.ReleaseVersion, 'InProgress' -PassThru - Write-Verbose -Verbose "Kicked off vPack build: $($buildInvocationInfo.WebUrl)" - $status = $buildInvocationInfo | Wait-AzDOBuildStatus -Status Completed -timeoutMinutes 60 - if ($status.result -ne 'Succeeded') { - throw "There are errors in snap build!!" - } - else { - $buildInvocationInfo | Remove-AzDOBuildTag -tag 'InProgress' -Pass | Add-AzDOBuildTag -tag 'SignedOff' - } - } - else { - Write-Verbose -Verbose "This is a preview release with version: $semanticVersion skipping releasing vPack" - } - -- stage: ReleaseDeps - dependsOn: GitHubTasks - displayName: Update pwsh.deps.json links - jobs: - - template: templates/release-UpdateDepsJson.yml - -- stage: ReleaseClose - displayName: Finish Release - dependsOn: - - ReleaseVPack - - BuildInfoJson - - UpdateDotnetDocker - - ReleaseDocker - - ReleaseSnap - - ChangesToMaster - - ReleaseDeps - jobs: - - template: templates/release/approvalJob.yml - parameters: - displayName: Retain Build - jobName: RetainBuild - instructions: | - Retain the build - - - template: templates/release/approvalJob.yml - parameters: - displayName: Delete release branch - jobName: DeleteBranch - instructions: | - Delete release diff --git a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml deleted file mode 100644 index dd9252a406f..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml +++ /dev/null @@ -1,63 +0,0 @@ -parameters: - ReleaseTagVar: v6.2.0 - ReleaseTagVarName: ReleaseTagVar - CreateJson: 'no' - UseJson: 'yes' - -steps: -- ${{ if eq(parameters['UseJson'],'yes') }}: - - task: DownloadBuildArtifacts@0 - inputs: - artifactName: 'BuildInfoJson' - itemPattern: '**/*.json' - downloadPath: '$(System.ArtifactsDirectory)' - displayName: Download Build Info Json - -- powershell: | - $path = "./build.psm1" - - if($env:REPOROOT){ - Write-Verbose "reporoot already set to ${env:REPOROOT}" -Verbose - exit 0 - } - - if(Test-Path -Path $path) - { - Write-Verbose "reporoot detect at: ." -Verbose - $repoRoot = '.' - } - else{ - $path = "./PowerShell/build.psm1" - if(Test-Path -Path $path) - { - Write-Verbose "reporoot detect at: ./PowerShell" -Verbose - $repoRoot = './PowerShell' - } - } - if($repoRoot) { - $vstsCommandString = "vso[task.setvariable variable=repoRoot]$repoRoot" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - } else { - Write-Verbose -Verbose "repo not found" - } - displayName: 'Set repo Root' - -- powershell: | - $createJson = ("${{ parameters.CreateJson }}" -ne "no") - $releaseTag = & "$env:REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson - $version = $releaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - - $azureVersion = $releaseTag.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: 'Set ${{ parameters.ReleaseTagVarName }} and other version Variables' - -- powershell: | - Get-ChildItem -Path env: - displayName: Capture environment - condition: succeededOrFailed() diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml deleted file mode 100644 index af6451004e4..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ /dev/null @@ -1,51 +0,0 @@ -jobs: -- job: DeleteBlob - variables: - - name: runCodesignValidationInjection - value : false - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: Azure Blob variable group - displayName: Delete blob is exists - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - steps: - - checkout: self - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - UseJson: no - - - task: AzurePowerShell@4 - displayName: Check if blob exists and delete if specified - inputs: - azureSubscription: '$(AzureFileCopySubscription)' - scriptType: inlineScript - azurePowerShellVersion: latestVersion - inline: | - try { - $container = Get-AzStorageContainer -Container '$(AzureVersion)' -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -ErrorAction Stop - - if ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'false') { - throw 'Azure blob container $(AzureVersion) already exists. To overwrite, use ForceAzureBlobDelete parameter' - } - elseif ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'true') { - Write-Verbose -Verbose 'Removing container $(AzureVersion) due to ForceAzureBlobDelete parameter' - Remove-AzStorageContainer -Name '$(AzureVersion)' -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -Force - } - } - catch { - if ($_.FullyQualifiedErrorId -eq 'ResourceNotFoundException,Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet.GetAzureStorageContainerCommand') { - Write-Verbose -Verbose 'Container "$(AzureVersion)" does not exists.' - } - else { - throw $_ - } - } - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml deleted file mode 100644 index 352458390f9..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml +++ /dev/null @@ -1,19 +0,0 @@ -parameters: - nativePathRoot: '' - -steps: - - powershell: | - $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar - $nativePath = "${{parameters.nativePathRoot }}${dirSeparatorChar}PowerShell" - Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" - - if ((Test-Path "$nativePath")) { - Remove-Item -Path "$nativePath" -Force -Recurse -Verbose -ErrorAction ignore - } - else { - Write-Verbose -Verbose -Message "No cleanup required." - } - - git clone --quiet $env:REPOROOT $nativePath - displayName: Clone PowerShell Repo to /PowerShell - errorActionPreference: silentlycontinue diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml deleted file mode 100644 index 0a416389bf4..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ /dev/null @@ -1,124 +0,0 @@ -parameters: - parentJobs: [] - -jobs: -- job: compliance - variables: - - name: runCodesignValidationInjection - value : false - - name: NugetSecurityAnalysisWarningLevel - value: none - - displayName: Compliance - dependsOn: - ${{ parameters.parentJobs }} - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: self - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifacts' - inputs: - buildType: current - downloadType: single - artifactName: results - downloadPath: '$(System.ArtifactsDirectory)' - - - powershell: | - dir "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture artifacts directory' - continueOnError: true - - - template: expand-compliance.yml - parameters: - architecture: fxdependent - version: $(version) - - - template: expand-compliance.yml - parameters: - architecture: x86 - version: $(version) - - - template: expand-compliance.yml - parameters: - architecture: x64 - version: $(version) - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 - displayName: 'Run BinSkim ' - inputs: - InputType: Basic - AnalyzeTarget: '$(CompliancePath)\*.dll;$(CompliancePath)\*.exe' - AnalyzeSymPath: 'SRV*' - AnalyzeVerbose: true - AnalyzeHashes: true - AnalyzeStatistics: true - continueOnError: true - - # add RoslynAnalyzers - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-autoapplicability.AutoApplicability@1 - displayName: 'Run AutoApplicability' - inputs: - ExternalRelease: true - IsSoftware: true - DataSensitivity: lbi - continueOnError: true - - # add codeMetrics - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-vulnerabilityassessment.VulnerabilityAssessment@0 - displayName: 'Run Vulnerability Assessment' - continueOnError: true - - # FXCop is not applicable - - # PreFASt is not applicable - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Security Analysis Logs to Build Artifacts' - continueOnError: true - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 - displayName: 'TSA upload to Codebase: PowerShellCore_201906' - inputs: - tsaVersion: TsaV2 - codeBaseName: 'PowerShellCore_201906' - uploadAPIScan: false - uploadBinSkim: true - uploadCredScan: false - uploadFortifySCA: false - uploadFxCop: false - uploadModernCop: false - uploadPoliCheck: false - uploadPREfast: false - uploadRoslyn: false - uploadTSLint: false - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 - displayName: 'Create Security Analysis Report' - inputs: - TsvFile: false - APIScan: false - BinSkim: true - CredScan: true - PoliCheck: true - PoliCheckBreakOn: Severity2Above - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)\tools' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml b/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml deleted file mode 100644 index 1b4f9067266..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml +++ /dev/null @@ -1,180 +0,0 @@ -jobs: - - job: APIScan - variables: - - name: runCodesignValidationInjection - value : false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: ReleaseTagVar - value: fromBranch - # Defines the variables APIScanClient, APIScanTenant and APIScanSecret - - group: PS-PS-APIScan - # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. - # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. - - group: symbols - - name: branchCounterKey - value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - - name: branchCounter - value: $[counter(variables['branchCounterKey'], 1)] - - group: DotNetPrivateBuildAccess - - group: Azure Blob variable group - - group: ReleasePipelineSecrets - - group: mscodehub-feed-read-general - - group: mscodehub-feed-read-akv - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - # APIScan can take a long time - timeoutInMinutes: 180 - - steps: - - template: ../SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - UseJson: no - - - template: ../insert-nuget-config-azfeed.yml - parameters: - repoRoot: '$(Build.SourcesDirectory)' - - - pwsh: | - Import-Module .\build.psm1 -force - Start-PSBootstrap - workingDirectory: '$(Build.SourcesDirectory)' - retryCountOnTaskFailure: 2 - displayName: 'Bootstrap' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - Import-Module .\build.psm1 -force - Find-DotNet - dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol - $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName - Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath" - displayName: Install dotnet-symbol - retryCountOnTaskFailure: 2 - - - pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - Install-AzCopy - displayName: Install AzCopy - retryCountOnTaskFailure: 2 - - - pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - $azcopy = Find-AzCopy - Write-Verbose -Verbose "Found AzCopy: $azcopy" - - $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)/winverify-symbols' -Force - Write-Host "##vso[task.setvariable variable=winverifySymbolsPath]$winverifySymbolsPath" - - & $azcopy cp https://$(StorageAccount).blob.core.windows.net/winverify-private $winverifySymbolsPath --recursive - - Get-ChildItem $winverifySymbolsPath -Recurse | Out-String | Write-Verbose -Verbose - - displayName: Download winverify-private Artifacts - retryCountOnTaskFailure: 2 - env: - AZCOPY_AUTO_LOGIN_TYPE: MSI - - - pwsh: | - Import-Module .\build.psm1 -force - Find-DotNet - Start-PSBuild -Configuration StaticAnalysis -PSModuleRestore -Clean -Runtime fxdependent-win-desktop - - $OutputFolder = Split-Path (Get-PSOutput) - Write-Host "##vso[task.setvariable variable=BinDir]$OutputFolder" - - Write-Verbose -Verbose -Message "Deleting ref folder from output folder" - if (Test-Path $OutputFolder/ref) { - Remove-Item -Recurse -Force $OutputFolder/ref - } - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'Build PowerShell Source' - - - pwsh: | - Get-ChildItem -Path env: - displayName: Capture Environment - condition: succeededOrFailed() - - # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet. - - pwsh: | - Import-Module .\build.psm1 -force - Find-DotNet - $pat = '$(SymbolServerPAT)' - if ($pat -like '*PAT*' -or $pat -eq '') - { - throw 'No PAT defined' - } - $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv' - $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:BinDir\*" --recurse-subdirectories - displayName: 'Download Symbols for binaries' - retryCountOnTaskFailure: 2 - workingDirectory: '$(Build.SourcesDirectory)' - - - pwsh: | - Get-ChildItem '$(BinDir)' -File -Recurse | - Foreach-Object { - [pscustomobject]@{ - Path = $_.FullName - Version = $_.VersionInfo.FileVersion - Md5Hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash - Sha512Hash = (Get-FileHash -Algorithm SHA512 -Path $_.FullName).Hash - } - } | Export-Csv -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' - displayName: 'Create release file hash artifact' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Build File Hash artifact' - inputs: - pathToPublish: '$(Build.SourcesDirectory)/ReleaseFileHash.csv' - artifactName: ReleaseFilesHash - retryCountOnTaskFailure: 2 - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@2 - displayName: 'Run APIScan' - inputs: - softwareFolder: '$(BinDir)' - softwareName: PowerShell - softwareVersionNum: '$(ReleaseTagVar)' - isLargeApp: false - preserveTempFiles: false - verbosityLevel: standard - # write a status update every 5 minutes. Default is 1 minute - statusUpdateInterval: '00:05:00' - env: - AzureServicesAuthConnectionString: RunAs=App - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@2 - continueOnError: true - displayName: 'Guardian Export' - inputs: - GdnExportVstsConsole: true - GdnExportSarifFile: true - GdnExportHtmlFile: true - GdnExportAllTools: false - GdnExportGdnToolApiScan: true - #this didn't do anything GdnExportCustomLogsFolder: '$(Build.ArtifactStagingDirectory)/Guardian' - - - task: TSAUpload@2 - displayName: 'TSA upload' - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.SourcesDirectory)\tools\guardian\tsaconfig-APIScan.json' - - - pwsh: | - Get-ChildItem -Path env: - displayName: Capture Environment - condition: succeededOrFailed() - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3 - displayName: 'Publish Guardian Artifacts' - inputs: - AllTools: false - APIScan: true - ArtifactName: APIScan diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml deleted file mode 100644 index 8db52fc83f0..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml +++ /dev/null @@ -1,83 +0,0 @@ -parameters: - - name: parentJobs - type: jobList - -jobs: -- job: compliance - variables: - - name: runCodesignValidationInjection - value : false - - name: NugetSecurityAnalysisWarningLevel - value: none - - # Defines the variables APIScanClient, APIScanTenant and APIScanSecret - - group: PS-PS-APIScan - - displayName: Compliance - dependsOn: - ${{ parameters.parentJobs }} - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - - # APIScan can take a long time - timeoutInMinutes: 180 - - steps: - - checkout: self - clean: true - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 - displayName: 'Run CredScan' - inputs: - suppressionsFile: tools/credScan/suppress.json - debugMode: false - continueOnError: true - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2 - displayName: 'Run PoliCheck' - inputs: - # targetType F means file or folder and is the only applicable value and the default - targetType: F - # 1 to enable source code comment scanning, which is what we should do for open source - optionsFC: 1 - # recurse - optionsXS: 1 - # run for severity 1, 2, 3 and 4 issues - optionsPE: '1|2|3|4' - # disable history management - optionsHMENABLE: 0 - # Excluclusion access database - optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' - # Terms Exclusion xml file - optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml - continueOnError: true - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3 - displayName: 'Publish Security Analysis Logs to Build Artifacts' - continueOnError: true - - - task: TSAUpload@2 - displayName: 'TSA upload' - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.SourcesDirectory)\tools\guardian\tsaconfig-others.json' - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 - displayName: 'Create Security Analysis Report' - inputs: - TsvFile: false - APIScan: false - BinSkim: false - CredScan: true - PoliCheck: true - PoliCheckBreakOn: Severity2Above - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)\tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml deleted file mode 100644 index 3e91b9174d2..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml +++ /dev/null @@ -1,90 +0,0 @@ -parameters: - - name: parentJobs - type: jobList - -jobs: -- job: generateNotice - variables: - - name: runCodesignValidationInjection - value : false - - name: NugetSecurityAnalysisWarningLevel - value: none - - displayName: Generate Notice - dependsOn: - ${{ parameters.parentJobs }} - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - timeoutInMinutes: 15 - - steps: - - checkout: self - clean: true - - - pwsh: | - [string]$Branch=$env:BUILD_SOURCEBRANCH - $branchOnly = $Branch -replace '^refs/heads/'; - $branchOnly = $branchOnly -replace '[_\-]' - - if ($branchOnly -eq 'master') { - $container = 'tpn' - } else { - $branchOnly = $branchOnly -replace '[\./]', '-' - $container = "tpn-$branchOnly" - } - - $vstsCommandString = "vso[task.setvariable variable=tpnContainer]$container" - Write-Verbose -Message $vstsCommandString -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Set ContainerName - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)\tools' - - - pwsh: | - ./tools/clearlyDefined/ClearlyDefined.ps1 -TestAndHarvest - displayName: Verify that packages have license data - - - task: msospo.ospo-extension.8d7f9abb-6896-461d-9e25-4f74ed65ddb2.notice@0 - displayName: 'NOTICE File Generator' - inputs: - outputfile: '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' - # output format can be html or text - outputformat: text - # this isn't working - # additionaldata: $(Build.SourcesDirectory)\assets\additionalAttributions.txt - - - - pwsh: | - Get-Content -Raw -Path $(Build.SourcesDirectory)\assets\additionalAttributions.txt | Out-File '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force -Append - Get-Content -Raw -Path '$(Build.SourcesDirectory)\assets\additionalAttributions.txt' - displayName: Append Additional Attributions - continueOnError: true - - - pwsh: | - Get-Content -Raw -Path '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' - displayName: Capture Notice - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload Notice' - inputs: - SourcePath: $(System.ArtifactsDirectory)\ThirdPartyNotices.txt - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: $(tpnContainer) - resourceGroup: '$(StorageResourceGroup)' - retryCountOnTaskFailure: 2 - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(System.ArtifactsDirectory) - artifactName: notice - displayName: Publish notice artifacts - retryCountOnTaskFailure: 2 diff --git a/tools/releaseBuild/azureDevOps/templates/expand-compliance.yml b/tools/releaseBuild/azureDevOps/templates/expand-compliance.yml deleted file mode 100644 index 4cc25433262..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/expand-compliance.yml +++ /dev/null @@ -1,12 +0,0 @@ -parameters: - architecture: x86 - version: 6.2.0 - -steps: - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-${{ parameters.version }}-symbols-win-${{ parameters.architecture }}.zip" -Destination "$(Build.StagingDirectory)\symbols\${{ parameters.architecture }}" - displayName: Expand symbols zip - ${{ parameters.architecture }} - - - powershell: | - tools/releaseBuild/createComplianceFolder.ps1 -ArtifactFolder "$(Build.StagingDirectory)\symbols\${{ parameters.architecture }}" -VSTSVariableName 'CompliancePath' - displayName: Expand Compliance file - ${{ parameters.architecture }} diff --git a/tools/releaseBuild/azureDevOps/templates/global-tool-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/global-tool-pkg-sbom.yml deleted file mode 100644 index d7200809cca..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/global-tool-pkg-sbom.yml +++ /dev/null @@ -1,64 +0,0 @@ -parameters: - - name: PackageVersion - - name: LinuxBinPath - - name: WindowsBinPath - - name: WindowsDesktopBinPath - - name: AlpineBinPath - - name: DestinationPath - - name: ListOfPackageTypes - type: object - default: - - Unified - - PowerShell.Linux.Alpine - - PowerShell.Linux.x64 - - PowerShell.Linux.arm32 - - PowerShell.Linux.arm64 - - PowerShell.Windows.x64 - -steps: - -- pwsh: | - Write-Verbose -Verbose 'LinuxBinPath path: ${{ parameters.LinuxBinPath }}' - Write-Verbose -Verbose 'WindowsBinPath path: ${{ parameters.WindowsBinPath }}' - Write-Verbose -Verbose 'WindowsDesktopBinPath path: ${{ parameters.WindowsDesktopBinPath }}' - Write-Verbose -Verbose 'AlpineBinPath path: ${{ parameters.AlpineBinPath }}' - - Import-Module -Name $env:REPOROOT\build.psm1 - Import-Module -Name $env:REPOROOT\tools\packaging - Start-PrepForGlobalToolNupkg -LinuxBinPath '${{ parameters.LinuxBinPath }}' -WindowsBinPath '${{ parameters.WindowsBinPath }}' -WindowsDesktopBinPath '${{ parameters.WindowsDesktopBinPath }}' -AlpineBinPath '${{ parameters.AlpineBinPath }}' - displayName: 'Preparation for Global Tools package creation.' - -# NOTE: The Unified package must always be created first, and so must always be first in ListOfPackageTypes. -- ${{ each value in parameters.ListOfPackageTypes }}: - - pwsh: | - $PackageType = '${{ value }}' - - Write-Verbose -Verbose "PackageType: $PackageType" - Write-Verbose -Verbose 'Destination path: ${{ parameters.PackagePath }}' - - # Create global tool NuSpec source for package. - Import-Module -Name $env:REPOROOT\build.psm1 - Import-Module -Name $env:REPOROOT\tools\packaging - New-GlobalToolNupkgSource -PackageType $PackageType -PackageVersion '${{ parameters.PackageVersion }}' -LinuxBinPath '${{ parameters.LinuxBinPath }}' -WindowsBinPath '${{ parameters.WindowsBinPath }}' -WindowsDesktopBinPath '${{ parameters.WindowsDesktopBinPath }}' -AlpineBinPath '${{ parameters.AlpineBinPath }}' - displayName: 'Create global tool NuSpec source for package.' - - - pwsh: | - Get-ChildItem -Path env: - displayName: 'Capture environment variables after Global Tool package source is created.' - - # NOTE: The above 'New-GlobalToolNupkgSource' task function sets the 'GlobalToolNuSpecSourcePath', 'GlobalToolPkgName', - # and 'GlobalToolCGManifestPath' environment variables. - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: $(GlobalToolNuSpecSourcePath) - Build_Repository_Uri: 'https://github.com/powershell/powershell' - PackageName: $(GlobalToolPkgName) - PackageVersion: ${{ parameters.PackageVersion }} - sourceScanPath: $(GlobalToolCGManifestPath) - displayName: SBOM for Global Tool package - - - pwsh: | - Import-Module -Name $env:REPOROOT\build.psm1 - Import-Module -Name $env:REPOROOT\tools\packaging - New-GlobalToolNupkgFromSource -PackageNuSpecPath "$env:GlobalToolNuSpecSourcePath" -PackageName "$env:GlobalToolPkgName" -DestinationPath '${{ parameters.DestinationPath }}' -CGManifestPath "$env:GlobalToolCGManifestPath" - displayName: 'Create global tool NuSpec package from NuSpec source.' diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml deleted file mode 100644 index 61b9df6c342..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +++ /dev/null @@ -1,8 +0,0 @@ -parameters: -- name: "repoRoot" - default: $(REPOROOT) -steps: - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self - parameters: - repoRoot: $(REPOROOT) - diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml deleted file mode 100644 index 48a50e0bf14..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ /dev/null @@ -1,57 +0,0 @@ -parameters: - parentJobs: [] - -jobs: -- job: json - variables: - - name: runCodesignValidationInjection - value : false - - name: NugetSecurityAnalysisWarningLevel - value: none - displayName: Create Json for Blob - dependsOn: - ${{ parameters.parentJobs }} - condition: succeeded() - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - #- task: @ - # inputs: - # - # displayName: '' - - checkout: self - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - - - task: AzureFileCopy@4 - displayName: 'upload daily-build-info JSON file to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(BuildInfoPath)' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: 'BuildInfo' - condition: and(succeeded(), eq(variables['IS_DAILY'], 'true')) - - - task: AzureCLI@1 - displayName: 'Make blob public' - inputs: - azureSubscription: '$(AzureFileCopySubscription)' - scriptLocation: inlineScript - inlineScript: 'az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion) --public-access blob' - condition: and(succeeded(), eq(variables['IS_DAILY'], 'true')) - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)\tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/linux-authenticode-sign.yml b/tools/releaseBuild/azureDevOps/templates/linux-authenticode-sign.yml deleted file mode 100644 index 719ba1a6c30..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/linux-authenticode-sign.yml +++ /dev/null @@ -1,184 +0,0 @@ -jobs: -- job: sign_linux_builds - displayName: Sign all linux builds - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - dependsOn: ['build_fxdependent', 'build_rpm'] - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: ESRP - - steps: - - checkout: self - clean: true - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuild.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Download deb build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildMinSize.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Download min-size build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildArm32.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Download arm32 build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildArm64.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Download arm64 build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshMarinerBuildAmd64.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Download mariner build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshMarinerBuildArm64.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Download mariner arm64 build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildAlpine.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildAlpine.tar.gz - displayName: Download alpine build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildAlpine.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars/pwshAlpineFxdBuildAmd64.tar.gz - displayName: Download alpine fxdependent build - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildFxdependent.tar.gz - path: $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildFxdependent.tar.gz - displayName: Download fxdependent build - - - pwsh: | - Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/linuxTars - displayName: Capture downloaded tars - - - pwsh: | - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuild" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuild - Write-Verbose -Verbose "File permisions after expanding" - Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild/pwsh | Select-Object -Property 'unixmode', 'size', 'name' - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildMinSize.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildMinSize.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm32.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm32.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32 - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64 - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshMarinerBuildAmd64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshMarinerBuildAmd64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64 - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshMarinerBuildArm64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshMarinerBuildArm64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64 - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildAlpine.tar.gz/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildAlpine.tar.gz/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshAlpineFxdBuildAmd64.tar.gz/pwshAlpineFxdBuildAmd64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpineFxd" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpineFxd -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshAlpineFxdBuildAmd64.tar.gz/pwshAlpineFxdBuildAmd64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpineFxd - - Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildFxdependent.tar.gz/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildFxdependent.tar.gz/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent - displayName: Expand builds - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: cloneToOfficialPath.yml - - - template: insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(PowerShellRoot) - - - pwsh: | - Set-Location $env:POWERSHELLROOT - import-module "$env:POWERSHELLROOT/build.psm1" - Sync-PSTags -AddRemoteIfMissing - displayName: SyncTags - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - - - checkout: ComplianceRepo - clean: true - - - template: shouldSign.yml - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuild - buildPrefixName: 'PowerShell Linux' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuildMinSize - buildPrefixName: 'PowerShell Linux Minimum Size' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuildArm32 - buildPrefixName: 'PowerShell Linux Arm32' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuildArm64 - buildPrefixName: 'PowerShell Linux Arm64' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshMarinerBuildAmd64 - buildPrefixName: 'PowerShell Linux x64 (Mariner) Framework Dependent' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshMarinerBuildArm64 - buildPrefixName: 'PowerShell Linux arm64 (Mariner) Framework Dependent' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuildAlpine - buildPrefixName: 'PowerShell Linux Alpine x64' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuildAlpineFxd - buildPrefixName: 'PowerShell Linux Alpine Fxd x64' - - - template: signBuildFiles.yml - parameters: - binLocation: pwshLinuxBuildFxdependent - buildPrefixName: 'PowerShell Linux Framework Dependent' diff --git a/tools/releaseBuild/azureDevOps/templates/linux-packaging.yml b/tools/releaseBuild/azureDevOps/templates/linux-packaging.yml deleted file mode 100644 index 59db37c64ac..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/linux-packaging.yml +++ /dev/null @@ -1,489 +0,0 @@ -parameters: - buildName: '' - uploadDisplayName: 'Upload' - -jobs: -- job: pkg_${{ parameters.buildName }} - displayName: Package ${{ parameters.buildName }} - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMSUbuntu20.04-Secure - variables: - - name: runCodesignValidationInjection - value: false - - name: build - value: ${{ parameters.buildName }} - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: ESRP - - group: DotNetPrivateBuildAccess - - steps: - - ${{ if or(eq(variables.build,'deb'), eq(variables.build,'rpm')) }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuild-signed - pattern: '**/pwshLinuxBuild.tar.gz' - displayName: Download deb build - - - ${{ if eq(variables.build,'deb') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize-signed - pattern: '**/pwshLinuxBuildMinSize.tar.gz' - displayName: Download min-size build - - - ${{ if eq(variables.build,'deb') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32-signed - pattern: '**/pwshLinuxBuildArm32.tar.gz' - displayName: Download arm32 build - - - ${{ if eq(variables.build,'deb') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64-signed - pattern: '**/pwshLinuxBuildArm64.tar.gz' - displayName: Download arm64 build - - - ${{ if eq(variables.build,'rpm') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64-signed - pattern: '**/pwshMarinerBuildAmd64.tar.gz' - displayName: Download mariner amd64 build - - - ${{ if eq(variables.build,'rpm') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64-signed - pattern: '**/pwshMarinerBuildArm64.tar.gz' - displayName: Download mariner arm64 build - - - ${{ if eq(variables.build,'alpine') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine-signed - pattern: '**/pwshLinuxBuildAlpine.tar.gz' - displayName: Download alpine build - - - ${{ if eq(variables.build,'alpine') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64-signed - pattern: '**/pwshAlpineFxdBuildAmd64.tar.gz' - displayName: Download alpine framework dependent build - - - ${{ if eq(variables.build,'fxdependent') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: authenticode-signed - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent-signed - pattern: '**/pwshLinuxBuildFxdependent.tar.gz' - displayName: Download fxdependent build - - - ${{ if or(eq(variables.build,'deb'), eq(variables.build,'rpm')) }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuild-meta - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuild-meta - displayName: Download deb build meta - - - ${{ if eq(variables.build,'deb') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildMinSize-meta - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize-meta - displayName: Download min-size build meta - - - ${{ if eq(variables.build,'deb') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildArm32-meta - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32-meta - displayName: Download arm32 build meta - - - ${{ if eq(variables.build,'deb') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildArm64-meta - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64-meta - displayName: Download arm64 build meta - - - ${{ if eq(variables.build,'rpm') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshMarinerBuildAmd64-meta - path: $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64-meta - displayName: Download mariner x64 build meta - - - ${{ if eq(variables.build,'rpm') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshMarinerBuildArm64-meta - path: $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64-meta - displayName: Download mariner arm64 build meta - - - ${{ if eq(variables.build,'alpine') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildAlpine-meta - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuild-meta - displayName: Download alpine build meta - - - ${{ if eq(variables.build,'alpine') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshAlpineFxdBuildAmd64-meta - path: $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64-meta - displayName: Download alpine build meta - - - ${{ if eq(variables.build,'fxdependent') }} : - - task: DownloadPipelineArtifact@2 - inputs: - artifact: pwshLinuxBuildFxdependent-meta - path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuild-meta - displayName: Download fxdependent build meta - - - pwsh: | - Get-ChildItem '$(Build.ArtifactStagingDirectory)' | Select-Object -Property 'unixmode', 'size', 'name' - displayName: Capture downloads - - - pwsh: | - if ('$(build)' -eq 'deb' -or '$(build)' -eq 'rpm') { - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshLinuxBuild-signed/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuild" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshLinuxBuild-signed/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuild - } - - if ('$(build)' -eq 'deb') { - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize-signed/pwshLinuxBuildMinSize.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize-signed/pwshLinuxBuildMinSize.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize - - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32-signed/pwshLinuxBuildArm32.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32-signed/pwshLinuxBuildArm32.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32 - - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64-signed/pwshLinuxBuildArm64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64-signed/pwshLinuxBuildArm64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64 - } - - if ('$(build)' -eq 'rpm') { - # for mariner x64 - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64-signed/pwshMarinerBuildAmd64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64-signed/pwshMarinerBuildAmd64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64 - - # for mariner arm64 - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64-signed/pwshMarinerBuildArm64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64-signed/pwshMarinerBuildArm64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshMarinerBuildArm64 - } - - if ('$(build)' -eq 'alpine') { - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine-signed/pwshLinuxBuildAlpine.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuild" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine-signed/pwshLinuxBuildAlpine.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuild - - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64-signed/pwshAlpineFxdBuildAmd64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64 -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64-signed/pwshAlpineFxdBuildAmd64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshAlpineFxdBuildAmd64 - } - - if ('$(build)' -eq 'fxdependent') { - Write-Verbose -Verbose "Expanding $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent-signed/pwshLinuxBuildFxdependent.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuild" - New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild -ItemType Directory - tar -xf $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent-signed/pwshLinuxBuildFxdependent.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuild - } - displayName: Expand all signed tar.gz - - - pwsh: | - Get-ChildItem '$(Build.ArtifactStagingDirectory)' | Select-Object -Property 'unixmode', 'size', 'name' - displayName: Capture expanded - - - checkout: self - clean: true - - - checkout: ComplianceRepo - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - pwsh: | - # create folder - sudo mkdir /PowerShell - - # make the current user the owner - sudo chown $env:USER /PowerShell - displayName: 'Create /PowerShell' - - - template: cloneToOfficialPath.yml - - - template: insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(PowerShellRoot) - - - powershell: | - import-module "$env:POWERSHELLROOT/build.psm1" - Sync-PSTags -AddRemoteIfMissing - displayName: SyncTags - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - - - powershell: | - Import-Module "$env:POWERSHELLROOT/build.psm1" - - Start-PSBootstrap -Package - displayName: 'Bootstrap' - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - powershell: | - try { - Import-Module "$env:POWERSHELLROOT/build.psm1" - Import-Module "$env:POWERSHELLROOT/tools/packaging" - - $metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json - - # LTSRelease.Package indicates that the release should be packaged as an LTS - $LTS = $metadata.LTSRelease.Package - Write-Verbose -Verbose -Message "LTS is set to: $LTS" - - Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' - - if ($LTS) { - Write-Verbose -Verbose "Packaging LTS" - Invoke-AzDevOpsLinuxPackageCreation -LTS -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' - } - } catch { - Get-Error - throw - } - displayName: 'Package' - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - - - powershell: | - $linuxPackages = Get-ChildItem "$env:POWERSHELLROOT/powershell*" -Include *.deb,*.rpm,*.tar.gz - - $bucket = 'release' - foreach ($linuxPackage in $linuxPackages) - { - $filePath = $linuxPackage.FullName - Write-Verbose "Publishing $filePath to $bucket" -Verbose - Write-Host "##vso[artifact.upload containerfolder=$bucket;artifactname=$bucket]$filePath" - } - displayName: Publish artifacts - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - retryCountOnTaskFailure: 2 - - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml - -- job: upload_${{ parameters.buildName }} - displayName: ${{ parameters.uploadDisplayName }} ${{ parameters.buildName }} - dependsOn: pkg_${{ parameters.buildName }} - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - variables: - - name: buildName - value: ${{ parameters.buildName }} - - group: ESRP - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: skipComponentGovernanceDetection - value: true - - steps: - - checkout: self - clean: true - - - checkout: ComplianceRepo - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - template: shouldSign.yml - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Deb Artifacts' - inputs: - downloadType: specific - itemPattern: '**/*.deb' - downloadPath: '$(System.ArtifactsDirectory)\finished' - condition: and(eq(variables['buildName'], 'DEB'), succeeded()) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download tar.gz Artifacts copy' - inputs: - downloadType: specific - itemPattern: '**/*.tar.gz' - downloadPath: '$(System.ArtifactsDirectory)\finished' - - - powershell: | - Write-Host 'We handle the min-size package only when uploading for deb build.' - Write-Host '- For deb build, the min-size package is moved to a separate folder "finished\minSize",' - Write-Host ' so that the min-size package can be uploaded to a different Az Blob container.' - Write-Host '- For other builds, the min-size package is removed after being downloaded, so that it' - Write-Host ' does not get accidentally uploaded to the wrong Az Blob container.' - - $minSizePkg = '$(System.ArtifactsDirectory)\finished\release\*-gc.tar.gz' - if (Test-Path -Path $minSizePkg) - { - if ('$(buildName)' -eq 'DEB') - { - $minSizeDir = '$(System.ArtifactsDirectory)\finished\minSize' - New-Item -Path $minSizeDir -Type Directory -Force > $null - Move-Item -Path $minSizePkg -Destination $minSizeDir - - Write-Host "`nCapture the min-size package moved to the target folder." - Get-ChildItem -Path $minSizeDir - } - else - { - Write-Host '$(buildName): Remove the min-size package.' - Remove-Item -Path $minSizePkg -Force - } - } - else - { - Write-Host 'min-size package not found, so skip this step.' - } - displayName: 'Move minSize package to separate folder' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download rpm Artifacts copy' - inputs: - downloadType: specific - itemPattern: '**/*.rpm' - downloadPath: '$(System.ArtifactsDirectory)\rpm' - condition: and(eq(variables['buildName'], 'RPM'), succeeded()) - - - template: EsrpScan.yml@ComplianceRepo - parameters: - scanPath: $(System.ArtifactsDirectory) - pattern: | - **\*.rpm - **\*.deb - **\*.tar.gz - - - ${{ if eq(variables['buildName'], 'RPM') }}: - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\rpm - signOutputPath: $(Build.StagingDirectory)\signedPackages - certificateId: "CP-450779-Pgp" - pattern: | - **\*.rh.*.rpm - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign RedHat RPM - OutputMode: AlwaysCopy - - - ${{ if eq(variables['buildName'], 'RPM') }}: - - template: EsrpSign.yml@ComplianceRepo - parameters: - # Sign in-place, previous task copied the files to this folder - buildOutputPath: $(Build.StagingDirectory)\signedPackages - signOutputPath: $(Build.StagingDirectory)\signedPackages - certificateId: "CP-459159-Pgp" - pattern: | - **\*.cm.*.rpm - **\*.cm?.*.rpm - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign Mariner RPM - OutputMode: NeverCopy - - # requires windows - - ${{ if ne(variables['buildName'], 'RPM') }}: - - task: AzureFileCopy@4 - displayName: 'Upload to Azure - DEB and tar.gz' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\finished\release\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - retryCountOnTaskFailure: 2 - - - template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\finished\release - - # requires windows - - task: AzureFileCopy@4 - displayName: 'Upload to Azure - min-size package for Guest Config' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\finished\minSize\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)-gc' - condition: and(eq(variables['buildName'], 'DEB'), succeeded()) - retryCountOnTaskFailure: 2 - - - template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\finished\minSize - condition: and(eq(variables['buildName'], 'DEB'), succeeded()) - - # requires windows - - task: AzureFileCopy@4 - displayName: 'Upload to Azure - RPM - Unsigned' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\rpm\release\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - retryCountOnTaskFailure: 2 - - # requires windows - - task: AzureFileCopy@4 - displayName: 'Upload to Azure - RPM - Signed' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\release\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - retryCountOnTaskFailure: 2 - - - template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\rpm\release - condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - template: upload-final-results.yml - parameters: - artifactPath: '$(Build.StagingDirectory)\signedPackages\release' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml deleted file mode 100644 index bb343bed54e..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ /dev/null @@ -1,313 +0,0 @@ -parameters: - buildName: '' - uploadDisplayName: 'Upload' - parentJob: '' - -jobs: -- job: build_${{ parameters.buildName }} - displayName: Build ${{ parameters.buildName }} - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMSUbuntu20.04-Secure - dependsOn: ${{ parameters.parentJob }} - variables: - - name: runCodesignValidationInjection - value: false - - name: build - value: ${{ parameters.buildName }} - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: ESRP - - group: DotNetPrivateBuildAccess - - steps: - - checkout: self - clean: true - - - checkout: ComplianceRepo - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - pwsh: | - # create folder - sudo mkdir /PowerShell - - # make the current user the owner - sudo chown $env:USER /PowerShell - displayName: 'Create /PowerShell' - - - template: cloneToOfficialPath.yml - - - template: insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(PowerShellRoot) - - - powershell: | - import-module "$env:POWERSHELLROOT/build.psm1" - Sync-PSTags -AddRemoteIfMissing - displayName: SyncTags - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - - - powershell: | - Import-Module "$env:POWERSHELLROOT/build.psm1" - - Start-PSBootstrap -Package - displayName: 'Bootstrap' - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - try { - Import-Module "$env:POWERSHELLROOT/build.psm1" - Import-Module "$env:POWERSHELLROOT/tools/packaging" - - Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' - - Write-Verbose -Verbose "File permisions after building" - Get-ChildItem -Path $(System.ArtifactsDirectory)/pwshLinuxBuild/pwsh | Select-Object -Property 'unixmode', 'size', 'name' - - } catch { - Get-Error - throw - } - displayName: 'Build' - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - workingDirectory: $(PowerShellRoot) - - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuild' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: ${{ parameters.buildName }} SBOM - PackageName: PowerShell Linux - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - ${{ if eq(variables.build,'rpm') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshMarinerBuildAmd64' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: Mariner x64 SBOM - PackageName: PowerShell Linux Framework Dependent - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - ${{ if eq(variables.build,'rpm') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshMarinerBuildArm64' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: Mariner arm64 SBOM - PackageName: PowerShell Linux Framework Dependent - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - ${{ if eq(variables.build,'deb') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: MinSize SBOM - PackageName: PowerShell Linux Minimum Size - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - ${{ if eq(variables.build,'deb') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: Arm32 SBOM - PackageName: PowerShell Linux Arm32 - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - ${{ if eq(variables.build,'deb') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: Arm64 SBOM - PackageName: PowerShell Linux Arm64 - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - ${{ if eq(variables.build,'alpine') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshAlpineFxdBuildAmd64' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: Alpine FXD SBOM - PackageName: PowerShell Alpine Framework Dependent AMD64 - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - pwsh: | - Set-Location '$(System.ArtifactsDirectory)/pwshLinuxBuild' - Write-Verbose -Verbose "File permisions before compressing" - Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild/pwsh | Select-Object -Property 'unixmode', 'size', 'name' - tar -czvf $(System.ArtifactsDirectory)/pwshLinuxBuild.tar.gz * - displayName: Compress pwshLinuxBuild - - - ${{ if eq(variables.build,'deb') }} : - - pwsh: | - Set-Location '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize' - tar -czvf $(System.ArtifactsDirectory)/pwshLinuxBuildMinSize.tar.gz * - Set-Location '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32' - tar -czvf $(System.ArtifactsDirectory)/pwshLinuxBuildArm32.tar.gz * - Set-Location '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64' - tar -czvf $(System.ArtifactsDirectory)/pwshLinuxBuildArm64.tar.gz * - displayName: Compress deb - - - ${{ if eq(variables.build,'rpm') }} : - - pwsh: | - Set-Location '$(System.ArtifactsDirectory)/pwshMarinerBuildAmd64' - tar -czvf $(System.ArtifactsDirectory)/pwshMarinerBuildAmd64.tar.gz * - displayName: Compress pwshMarinerBuildAmd64 - - - ${{ if eq(variables.build,'alpine') }} : - - pwsh: | - Set-Location '$(System.ArtifactsDirectory)/pwshAlpineFxdBuildAmd64' - tar -czvf $(System.ArtifactsDirectory)/pwshAlpineFxdBuildAmd64.tar.gz * - displayName: Compress pwshAlpineFxdBuildAmd64 - - - ${{ if eq(variables.build,'rpm') }} : - - pwsh: | - Set-Location '$(System.ArtifactsDirectory)/pwshMarinerBuildArm64' - tar -czvf $(System.ArtifactsDirectory)/pwshMarinerBuildArm64.tar.gz * - displayName: Compress pwshMarinerBuildArm64 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuild.tar.gz' - artifactName: pwshLinuxBuild.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuild-meta' - artifactName: pwshLinuxBuild-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize.tar.gz' - artifactName: pwshLinuxBuildMinSize.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize-meta' - artifactName: pwshLinuxBuildMinSize-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32.tar.gz' - artifactName: pwshLinuxBuildArm32.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32-meta' - artifactName: pwshLinuxBuildArm32-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64.tar.gz' - artifactName: pwshLinuxBuildArm64.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'deb') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64-meta' - artifactName: pwshLinuxBuildArm64-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'rpm') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshMarinerBuildAmd64.tar.gz' - artifactName: pwshMarinerBuildAmd64.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'rpm') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshMarinerBuildAmd64-meta' - artifactName: pwshMarinerBuildAmd64-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'rpm') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshMarinerBuildArm64.tar.gz' - artifactName: pwshMarinerBuildArm64.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'rpm') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshMarinerBuildArm64-meta' - artifactName: pwshMarinerBuildArm64-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'alpine') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuild.tar.gz' - artifactName: pwshLinuxBuildAlpine.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'alpine') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuild-meta' - artifactName: pwshLinuxBuildAlpine-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'alpine') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshAlpineFxdBuildAmd64.tar.gz' - artifactName: pwshAlpineFxdBuildAmd64.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'alpine') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshAlpineFxdBuildAmd64-meta' - artifactName: pwshAlpineFxdBuildAmd64-meta - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'fxdependent') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuild.tar.gz' - artifactName: pwshLinuxBuildFxdependent.tar.gz - retryCountOnTaskFailure: 2 - - - ${{ if eq(variables.build,'fxdependent') }} : - - task: PublishPipelineArtifact@1 - inputs: - path: '$(System.ArtifactsDirectory)/pwshLinuxBuild-meta' - artifactName: pwshLinuxBuildFxdependent-meta - retryCountOnTaskFailure: 2 diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml deleted file mode 100644 index 8159c2bc7d9..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ /dev/null @@ -1,121 +0,0 @@ -parameters: - buildArchitecture: 'x64' - -jobs: - - job: MacFileSigningJob_${{ parameters.buildArchitecture }} - displayName: macOS File signing ${{ parameters.buildArchitecture }} - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: ESRP - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: repoFolder - value: PowerShell - - name: repoRoot - value: $(Agent.BuildDirectory)\$(repoFolder) - - name: complianceRepoFolder - value: compliance - - steps: - - checkout: self - clean: true - path: $(repoFolder) - - - checkout: ComplianceRepo - clean: true - path: $(complianceRepoFolder) - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: shouldSign.yml - - - task: DownloadBuildArtifacts@0 - inputs: - artifactName: 'macosBinResults' - itemPattern: '**/*.zip' - downloadPath: '$(System.ArtifactsDirectory)\Symbols' - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture Downloaded Artifacts' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - pwsh: | - $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*${{ parameters.buildArchitecture }}*.zip' - Write-Verbose -Verbose "Zip Path: $zipPath" - - $expandedFolder = $zipPath.BaseName - Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" - Write-Host "##vso[task.setvariable variable=SymbolsFolder]$expandedFolder" - - Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force - displayName: Expand symbols zip - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture artifacts dir Binaries' - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\$(SymbolsFolder)" -Recurse -Include pwsh, *.dylib - displayName: 'Capture Expanded Binaries' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - pwsh: | - $null = new-item -type directory -path "$(Build.StagingDirectory)\macos" - $zipFile = "$(Build.StagingDirectory)\macos\powershell-files-$(Version)-osx-${{ parameters.buildArchitecture }}.zip" - Get-ChildItem "$(System.ArtifactsDirectory)\$(SymbolsFolder)" -Recurse -Include pwsh, *.dylib | - Compress-Archive -Destination $zipFile - Write-Host $zipFile - displayName: 'Compress macOS binary files' - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(Build.StagingDirectory)\macos - signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages - certificateId: "CP-401337-Apple" - pattern: | - **\*.zip - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign macOS Binaries - - - pwsh: | - $destination = "$(System.ArtifactsDirectory)\azureMacOs_${{ parameters.buildArchitecture }}" - New-Item -Path $destination -Type Directory - $zipPath = Get-ChildItem "$(Build.StagingDirectory)\signedMacOSPackages\powershell-*.zip" -Recurse | select-object -expandproperty fullname - foreach ($z in $zipPath) { Expand-Archive -Path $z -DestinationPath $destination } - displayName: 'Extract and copy macOS artifacts for upload' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\azureMacOs_${{ parameters.buildArchitecture }} - artifactFilter: "*" - artifactName: signedMacOsBins_${{ parameters.buildArchitecture }} - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - ${{ if eq(variables['SHOULD_SIGN'], 'true') }}: - - template: EsrpScan.yml@ComplianceRepo - parameters: - scanPath: $(System.ArtifactsDirectory)\azureMacOs_${{ parameters.buildArchitecture }} - pattern: | - **\* - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(repoRoot)\tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml deleted file mode 100644 index c853a21ef37..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ /dev/null @@ -1,143 +0,0 @@ -parameters: - parentJob: '' - buildArchitecture: x64 - -jobs: -- job: package_macOS_${{ parameters.buildArchitecture }} - displayName: Package macOS ${{ parameters.buildArchitecture }} - condition: succeeded() - pool: - vmImage: macos-latest - variables: - # Turn off Homebrew analytics - - name: HOMEBREW_NO_ANALYTICS - value: 1 - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: DotNetPrivateBuildAccess - steps: - - checkout: self - clean: true - - - pwsh: | - # create folder - sudo mkdir "$(Agent.TempDirectory)/PowerShell" - - # make the current user the owner - sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" - displayName: 'Create $(Agent.TempDirectory)/PowerShell' - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: shouldSign.yml - - - template: cloneToOfficialPath.yml - parameters: - nativePathRoot: '$(Agent.TempDirectory)' - - - task: DownloadBuildArtifacts@0 - displayName: Download macosBinResults - inputs: - artifactName: 'macosBinResults' - itemPattern: '**/*${{ parameters.buildArchitecture }}.zip' - downloadPath: '$(System.ArtifactsDirectory)/Symbols' - - - task: DownloadBuildArtifacts@0 - displayName: Download signedMacOsBins - inputs: - artifactName: 'signedMacOsBins_${{ parameters.buildArchitecture }}' - itemPattern: '**/*' - downloadPath: '$(System.ArtifactsDirectory)/macOsBins' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture Downloaded Artifacts' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - pwsh: | - $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*${{ parameters.buildArchitecture }}.zip' - Write-Verbose -Verbose "Zip Path: $zipPath" - - $expandedFolder = $zipPath.BaseName - Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" - Write-Host "##vso[task.setvariable variable=SymbolsFolder]$expandedFolder" - - Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force - displayName: Expand symbols zip - - - pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)/macOsBins/signedMacOsBins_${{ parameters.buildArchitecture }}/' - $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - displayName: Merge signed files with Build - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/$(SymbolsFolder)' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - PackageName: PowerShell macOS ${{ parameters.buildArchitecture }} - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - - pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - - $destFolder = '$(System.ArtifactsDirectory)\signedZip' - $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - - $null = New-Item -ItemType Directory -Path $destFolder -Force - - $BuildPackagePath = New-PSBuildZip -BuildPath $BuildPath -DestinationFolder $destFolder - - Write-Verbose -Verbose "New-PSSignedBuildZip returned `$BuildPackagePath as: $BuildPackagePath" - Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$BuildPackagePath" - - $vstsCommandString = "vso[task.setvariable variable=BuildPackagePath]$BuildPackagePath" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Compress signed files - retryCountOnTaskFailure: 2 - - - - pwsh: | - try { - tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap - } catch { - Get-Error - throw - } - displayName: 'Bootstrap VM' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - # Add -SkipReleaseChecks as a mitigation to unblock release. - # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. - try { - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks - } catch { - Get-Error - throw - } - displayName: 'Package' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(PowerShellRoot)/tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml deleted file mode 100644 index d4901580b0b..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ /dev/null @@ -1,135 +0,0 @@ -parameters: - buildArchitecture: x64 - -jobs: -- job: MacPackageSigningJob_${{ parameters.buildArchitecture }} - displayName: macOS Package signing ${{ parameters.buildArchitecture }} - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - variables: - - group: ESRP - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: repoFolder - value: PowerShell - - name: repoRoot - value: $(Agent.BuildDirectory)\$(repoFolder) - - name: complianceRepoFolder - value: compliance - - steps: - - checkout: self - clean: true - path: $(repoFolder) - - - checkout: ComplianceRepo - clean: true - path: $(complianceRepoFolder) - - - template: shouldSign.yml - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - task: DownloadBuildArtifacts@0 - inputs: - artifactName: 'macosPkgResults' - itemPattern: '**/*' - downloadPath: '$(System.ArtifactsDirectory)' - - - pwsh: | - dir "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture Downloaded Artifacts' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - pwsh: | - $null = new-item -type directory -path "$(Build.StagingDirectory)\macos" - $zipFile = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-${{ parameters.buildArchitecture }}.zip" - Compress-Archive -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-${{ parameters.buildArchitecture }}.pkg" -Destination $zipFile - Write-Host $zipFile - - $ltsPkgPath = "$(System.ArtifactsDirectory)\macosPkgResults\powershell-lts-$(Version)-osx-${{ parameters.buildArchitecture }}.pkg" - - if(Test-Path $ltsPkgPath) - { - $ltsZipFile = "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-${{ parameters.buildArchitecture }}.zip" - Compress-Archive -Path $ltsPkgPath -Destination $ltsZipFile - Write-Host $ltsZipFile - } - displayName: 'Compress macOS Package' - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(Build.StagingDirectory)\macos - signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages - certificateId: "CP-401337-Apple" - pattern: | - **\*.zip - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign pkg - - - template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\macosPkgResults - artifactFilter: "*${{ parameters.buildArchitecture }}.tar.gz" - - - pwsh: | - $destination = "$(System.ArtifactsDirectory)\azureMacOs" - New-Item -Path $destination -Type Directory - $zipPath = dir "$(Build.StagingDirectory)\signedMacOSPackages\powershell-*.zip" -Recurse | select-object -expandproperty fullname - foreach ($z in $zipPath) { Expand-Archive -Path $z -DestinationPath $destination } - $targzPath = dir "$(System.ArtifactsDirectory)\*osx*.tar.gz" -Recurse | select-object -expandproperty fullname - Copy-Item -Path $targzPath -Destination $destination - displayName: 'Extract and copy macOS artifacts for upload' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\azureMacOs - artifactFilter: "*.pkg" - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - pwsh: | - $null = new-item -type directory -path "$(Build.StagingDirectory)\macos-unsigned" - Copy-Item -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-x64.pkg" -Destination "$(Build.StagingDirectory)\macos-unsigned" - Copy-Item -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-x64.tar.gz" -Destination "$(Build.StagingDirectory)\macos-unsigned" - displayName: 'Create unsigned folder to upload' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - - - task: AzureFileCopy@4 - displayName: 'AzureBlob File Copy - unsigned' - inputs: - SourcePath: '$(Build.StagingDirectory)\macos-unsigned\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - retryCountOnTaskFailure: 2 - - - task: AzureFileCopy@4 - displayName: 'AzureBlob File Copy - signed' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\azureMacOs\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - retryCountOnTaskFailure: 2 - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(repoRoot)/tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml deleted file mode 100644 index d173e900434..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ /dev/null @@ -1,68 +0,0 @@ -parameters: - buildArchitecture: 'x64' - -jobs: -- job: build_macOS_${{ parameters.buildArchitecture }} - displayName: Build macOS ${{ parameters.buildArchitecture }} - condition: succeeded() - pool: - vmImage: macos-latest - variables: - # Turn off Homebrew analytics - - name: HOMEBREW_NO_ANALYTICS - value: 1 - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: DotNetPrivateBuildAccess - steps: - #- task: @ - # inputs: - # - # displayName: '' - - checkout: self - clean: true - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - pwsh: | - # create folder - sudo mkdir "$(Agent.TempDirectory)/PowerShell" - - # make the current user the owner - sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" - displayName: 'Create $(Agent.TempDirectory)/PowerShell' - - - template: cloneToOfficialPath.yml - parameters: - nativePathRoot: '$(Agent.TempDirectory)' - - - pwsh: | - tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap - displayName: 'Bootstrap VM' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(PowerShellRoot) - - - pwsh: | - $env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)' - # Add -SkipReleaseChecks as a mitigation to unblock release. - # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks - $env:AzDevOpsFeedPAT2 = $null - displayName: 'Build' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)/tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml deleted file mode 100644 index 0a0e3b96cc1..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ /dev/null @@ -1,139 +0,0 @@ -parameters: - - name: PackageVersion - - name: PackagePath - - name: WinFxdPath - - name: LinuxFxdPath - - name: ListOfFiles - type: object - default: - - Microsoft.Management.Infrastructure.CimCmdlets.dll - - Microsoft.PowerShell.Commands.Diagnostics.dll - - Microsoft.PowerShell.Commands.Management.dll - - Microsoft.PowerShell.Commands.Utility.dll - - Microsoft.PowerShell.ConsoleHost.dll - - Microsoft.PowerShell.CoreCLR.Eventing.dll - - Microsoft.PowerShell.Security.dll - - Microsoft.PowerShell.SDK.dll - - Microsoft.WSMan.Management.dll - - Microsoft.WSMan.Runtime.dll - - System.Management.Automation.dll - -steps: - -- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self - parameters: - repoRoot: $(REPOROOT) - -- pwsh: | - Import-Module "$env:REPOROOT/build.psm1" -Force - Start-PSBootstrap - - $sharedModules = @('Microsoft.PowerShell.Commands.Management', - 'Microsoft.PowerShell.Commands.Utility', - 'Microsoft.PowerShell.ConsoleHost', - 'Microsoft.PowerShell.Security', - 'System.Management.Automation' - ) - - $winOnlyModules = @('Microsoft.Management.Infrastructure.CimCmdlets', - 'Microsoft.PowerShell.Commands.Diagnostics', - 'Microsoft.PowerShell.CoreCLR.Eventing', - 'Microsoft.WSMan.Management', - 'Microsoft.WSMan.Runtime' - ) - - $refAssemblyFolder = Join-Path '$(System.ArtifactsDirectory)' 'RefAssembly' - $null = New-Item -Path $refAssemblyFolder -Force -Verbose -Type Directory - - Start-PSBuild -Clean -Runtime linux-x64 -Configuration Release - - $sharedModules | Foreach-Object { - $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net9.0\refint\$_.dll" - Write-Verbose -Verbose "RefAssembly: $refFile" - Copy-Item -Path $refFile -Destination "$refAssemblyFolder\$_.dll" -Verbose - $refDoc = "$env:REPOROOT\src\$_\bin\Release\net9.0\$_.xml" - if (-not (Test-Path $refDoc)) { - Write-Warning "$refDoc not found" - Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net9.0\" | Out-String | Write-Verbose -Verbose - } - else { - Copy-Item -Path $refDoc -Destination "$refAssemblyFolder\$_.xml" -Verbose - } - } - - Start-PSBuild -Clean -Runtime win7-x64 -Configuration Release - - $winOnlyModules | Foreach-Object { - $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net9.0\refint\*.dll" - Write-Verbose -Verbose 'RefAssembly: $refFile' - Copy-Item -Path $refFile -Destination "$refAssemblyFolder\$_.dll" -Verbose - $refDoc = "$env:REPOROOT\src\$_\bin\Release\net9.0\$_.xml" - if (-not (Test-Path $refDoc)) { - Write-Warning "$refDoc not found" - Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net9.0" | Out-String | Write-Verbose -Verbose - } - else { - Copy-Item -Path $refDoc -Destination "$refAssemblyFolder\$_.xml" -Verbose - } - } - - Get-ChildItem $refAssemblyFolder -Recurse | Out-String | Write-Verbose -Verbose - - # Set RefAssemblyPath path variable - $vstsCommandString = "vso[task.setvariable variable=RefAssemblyPath]${refAssemblyFolder}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - displayName: Build reference assemblies - env: - __DOTNET_RUNTIME_FEED: $(RUNTIME_SOURCEFEED) - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - -- ${{ each value in parameters.ListOfFiles }}: - - pwsh: | - $FileName = '${{ value }}' - $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) - $FilePackagePath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath $FileBaseName - $CGManifestPath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath 'CGManifest' - Write-Verbose -Verbose "FileName to package: $FileName" - Write-Verbose -Verbose "FilePackage path: $FilePackagePath" - Write-Verbose -Verbose "CGManifest path: $CGManifestPath" - # Set SBOM package name - $vstsCommandString = "vso[task.setvariable variable=SbomFilePackageName]${FileBaseName}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - # Set SBOM package path variable - $vstsCommandString = "vso[task.setvariable variable=SbomFilePackagePath]${FilePackagePath}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - # Set CGManifest path variable - $vstsCommandString = "vso[task.setvariable variable=CGManifestPath]${CGManifestPath}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - # Create Nuget package sources - Import-Module -Name $env:REPOROOT\build.psm1 - Import-Module -Name $env:REPOROOT\tools\packaging - Find-DotNet - New-ILNugetPackageSource -File $FileName -PackagePath '${{ parameters.PackagePath }}' -PackageVersion '${{ parameters.PackageVersion }}' -WinFxdBinPath '${{ parameters.WinFxdPath }}' -LinuxFxdBinPath '${{ parameters.LinuxFxdPath }}' -CGManifestPath $CGManifestPath -RefAssemblyPath $(RefAssemblyPath) - displayName: 'Create NuGet Package source for single file' - - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: $(SbomFilePackagePath) - Build_Repository_Uri: 'https://github.com/powershell/powershell' - PackageName: $(SbomFilePackageName) - PackageVersion: ${{ parameters.PackageVersion }} - sourceScanPath: $(CGManifestPath) - displayName: SBOM for NuGetPkg - - - pwsh: | - $FileName = '${{ value }}' - $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) - $FilePackagePath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath $FileBaseName - Write-Verbose -Verbose "FileName to package: $FileName" - Write-Verbose -Verbose "FilePackage path: $FilePackagePath" - Import-Module -Name $env:REPOROOT\build.psm1 - Import-Module -Name $env:REPOROOT\tools\packaging - Find-DotNet - New-ILNugetPackageFromSource -FileName $FileName -PackageVersion '${{ parameters.PackageVersion }}' -PackagePath '${{ parameters.PackagePath }}' - displayName: 'Create NuGet Package for single file' diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml deleted file mode 100644 index 22f791bf0eb..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ /dev/null @@ -1,290 +0,0 @@ -parameters: - parentJobs: [] - -jobs: -- job: build_nuget - dependsOn: - ${{ parameters.parentJobs }} - displayName: Build NuGet packages - condition: succeeded() - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - - timeoutInMinutes: 90 - - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: build - value: ${{ parameters.buildName }} - - group: ESRP - - name: GenAPIToolPath - value: '$(System.ArtifactsDirectory)/GenAPI' - - name: PackagePath - value: '$(System.ArtifactsDirectory)/UnifiedPackagePath' - - name: winFxdPath - value: '$(System.ArtifactsDirectory)/winFxd' - - name: winFxdWinDesktopPath - value: '$(System.ArtifactsDirectory)/winFxdWinDesktop' - - name: linuxFxdPath - value: '$(System.ArtifactsDirectory)/linuxFxd' - - name: alpineFxdPath - value: '$(System.ArtifactsDirectory)/alpineFxd' - - group: DotNetPrivateBuildAccess - - steps: - - checkout: self - clean: true - - - checkout: ComplianceRepo - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - powershell: | - $content = Get-Content "$env:REPOROOT/global.json" -Raw | ConvertFrom-Json - $vstsCommandString = "vso[task.setvariable variable=SDKVersion]$($content.sdk.version)" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Find SDK version from global.json' - - - pwsh: | - Import-Module "$env:REPOROOT/build.psm1" -Force - # We just need .NET but we fixed this in an urgent situation. - Start-PSBootStrap -Verbose - displayName: Bootstrap - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download PowerShell build artifacts - finalResults' - inputs: - buildType: current - downloadType: single - artifactName: finalResults - downloadPath: '$(System.ArtifactsDirectory)' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download PowerShell build artifacts - macosPkgResults' - inputs: - buildType: current - downloadType: single - artifactName: macosPkgResults - downloadPath: '$(System.ArtifactsDirectory)' - - - powershell: 'Get-ChildItem $(System.ArtifactsDirectory) -recurse' - displayName: 'Capture downloaded artifacts' - - - powershell: | - $packagePath = (Join-Path $(System.ArtifactsDirectory) packages) - New-Item $packagePath -ItemType Directory -Force > $null - $packages = Get-ChildItem $(System.ArtifactsDirectory) -Include *.zip, *.tar.gz -Recurse - $packages | ForEach-Object { Copy-Item $_.FullName -Destination $packagePath -Verbose } - Get-ChildItem $packagePath -Recurse - displayName: 'Conflate packages to same folder' - - - task: ExtractFiles@1 - displayName: 'Extract files win-fxdependent' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/PowerShell-*-win-fxdependent.zip' - destinationFolder: '$(winFxdPath)' - - - task: ExtractFiles@1 - displayName: 'Extract files win-fxdependentWinDesktop' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/PowerShell-*-win-fxdependentWinDesktop.zip' - destinationFolder: '$(winFxdWinDesktopPath)' - - - task: ExtractFiles@1 - displayName: 'Extract files linux-fxdependent' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/powershell-*-linux-x64-fxdependent.tar.gz' - destinationFolder: '$(linuxFxdPath)' - - - task: ExtractFiles@1 - displayName: 'Extract files alpine-fxdependent' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/powershell-*-linux-x64-musl-noopt-fxdependent.tar.gz' - destinationFolder: '$(alpineFxdPath)' - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: shouldSign.yml - - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - # Create nuget packages along with SBOM manifests. - - template: nuget-pkg-sbom.yml - parameters: - PackageVersion: $(Version) - PackagePath: $(PackagePath) - WinFxdPath: $(winFxdPath) - LinuxFxdPath: $(linuxFxdPath) - - - pwsh: | - Get-ChildItem $(linuxFxdPath) - Get-ChildItem $(winFxdPath) - Get-ChildItem $(winFxdWinDesktopPath) - Get-ChildItem $(alpineFxdPath) - displayName: Capture fxd folders - - # Create Global Tool packages along with SBOM manifests - - template: global-tool-pkg-sbom.yml - parameters: - PackageVersion: $(Version) - LinuxBinPath: $(linuxFxdPath) - WindowsBinPath: $(winFxdPath) - WindowsDesktopBinPath: $(winFxdWinDesktopPath) - AlpineBinPath: $(alpineFxdPath) - DestinationPath: $(PackagePath)\globaltool - - - pwsh: | - Get-ChildItem "$(PackagePath)" -Recurse - displayName: Capture generated packages - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(PackagePath) - signOutputPath: $(System.ArtifactsDirectory)\signed - certificateId: "CP-401405" - pattern: | - **\*.nupkg - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign NuPkg - - - pwsh: | - if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } - Copy-Item -Path '$(PackagePath)\*.nupkg' -Destination '$(System.ArtifactsDirectory)\signed\' -Verbose -Force - Copy-Item -Path '$(PackagePath)\globaltool\*.nupkg' -Destination '$(System.ArtifactsDirectory)\signed\' -Verbose -Force - displayName: Fake copy when not signing - condition: eq(variables['SHOULD_SIGN'], 'false') - - - pwsh: | - Import-Module "${env:REPOROOT}\build.psm1" -Force - Get-ChildItem -Recurse "$(System.ArtifactsDirectory)\signed\*.nupkg" -Verbose | ForEach-Object { Start-NativeExecution -sb { nuget.exe verify -All $_.FullName } } - displayName: Verify all packages are signed - condition: eq(variables['SHOULD_SIGN'], 'true') - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run MpCmdRun.exe' - inputs: - FileDirPath: '$(PackagePath)' - TreatStaleSignatureAs: Warning - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Security Analysis Logs' - - - template: upload-final-results.yml - parameters: - artifactPath: '$(System.ArtifactsDirectory)\signed' - - - pwsh: | - if (-not (Test-Path "$(System.ArtifactsDirectory)\signed\globaltool")) - { - $null = New-Item -Path "$(System.ArtifactsDirectory)\signed\globaltool" -ItemType Directory -Force - } - - Move-Item -Path "$(System.ArtifactsDirectory)\signed\PowerShell.*" -Destination "$(System.ArtifactsDirectory)\signed\globaltool" -Force - Get-ChildItem "$(System.ArtifactsDirectory)\signed\globaltool" -Recurse - displayName: Move global tool packages to subfolder and capture - - - pwsh: | - $packagePath = (Join-Path $(System.ArtifactsDirectory) checksum) - New-Item $packagePath -ItemType Directory -Force > $null - $srcPaths = @("$(System.ArtifactsDirectory)\finalResults", "$(System.ArtifactsDirectory)\macosPkgResults", "$(System.ArtifactsDirectory)\signed") - - $packages = Get-ChildItem -Path $srcPaths -Include *.zip, *.tar.gz, *.msi*, *.pkg, *.deb, *.rpm -Exclude "PowerShell-Symbols*" -Recurse - $packages | ForEach-Object { Copy-Item $_.FullName -Destination $packagePath -Verbose } - - $packagePathList = Get-ChildItem $packagePath -Recurse | Select-Object -ExpandProperty FullName | Out-String - Write-Verbose -Verbose $packagePathList - - $checksums = Get-ChildItem -Path $packagePath -Exclude "SHA512SUMS" | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA512).Hash.ToLower() - - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - - $checksums | Out-File -FilePath "$packagePath\SHA512SUMS" -Force - - - $fileContent = Get-Content -Path "$packagePath\SHA512SUMS" -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent - - Copy-Item -Path "$packagePath\SHA512SUMS" -Destination '$(System.ArtifactsDirectory)\signed\' -verbose - displayName: Generate checksum file for packages - - - pwsh: | - $packagePath = (Join-Path $(System.ArtifactsDirectory) checksum_gbltool) - New-Item $packagePath -ItemType Directory -Force > $null - $srcPaths = @("$(System.ArtifactsDirectory)\signed\globaltool") - $packages = Get-ChildItem -Path $srcPaths -Include *.nupkg -Recurse - $packages | ForEach-Object { Copy-Item $_.FullName -Destination $packagePath -Verbose } - - $packagePathList = Get-ChildItem $packagePath -Recurse | Select-Object -ExpandProperty FullName | Out-String - Write-Verbose -Verbose $packagePathList - - $checksums = Get-ChildItem -Path $packagePath -Exclude "SHA512SUMS" | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA512).Hash.ToLower() - - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - - $checksums | Out-File -FilePath "$packagePath\SHA512SUMS" -Force - - $fileContent = Get-Content -Path "$packagePath\SHA512SUMS" -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent - - Copy-Item -Path "$packagePath\SHA512SUMS" -Destination '$(System.ArtifactsDirectory)\signed\globaltool\' -verbose - displayName: Generate checksum for global tools - - - template: upload-final-results.yml - parameters: - artifactPath: '$(System.ArtifactsDirectory)\checksum' - artifactFilter: SHA512SUMS - - - task: AzureFileCopy@4 - displayName: 'Upload NuGet packages to Azure' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\*' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)-nuget' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - retryCountOnTaskFailure: 2 - - - task: AzureFileCopy@4 - displayName: 'Upload global tool packages to Azure' - inputs: - sourcePath: '$(System.ArtifactsDirectory)\signed\globaltool\*' - azureSubscription: '$(GlobalToolSubscription)' - Destination: AzureBlob - storage: '$(GlobalToolStorageAccount)' - ContainerName: 'tool-private' - blobPrefix: '$(Version)' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - retryCountOnTaskFailure: 2 - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(PackagePath)' diff --git a/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml b/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml deleted file mode 100644 index d183601a06c..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml +++ /dev/null @@ -1,102 +0,0 @@ -steps: -- checkout: self - clean: true - -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: BuildInfoJson - path: '$(Pipeline.Workspace)/releasePipeline/BuildInfoJson' - -- pwsh: | - Import-Module '$(Build.SourcesDirectory)/tools/ci.psm1' - $jsonFile = Get-Item "$ENV:PIPELINE_WORKSPACE/releasePipeline/BuildInfoJson/*.json" - $fileName = Split-Path $jsonFile -Leaf - - $dateTime = [datetime]::UtcNow - $dateTime = [datetime]::new($dateTime.Ticks - ($dateTime.Ticks % [timespan]::TicksPerSecond), $dateTime.Kind) - - $metadata = Get-Content ./tools/metadata.json | ConvertFrom-Json - $stableRelease = $metadata.StableRelease.Latest - $ltsRelease = $metadata.LTSRelease.Latest - - Write-Verbose -Verbose "Writing $jsonFile contents:" - $buildInfoJsonContent = Get-Content $jsonFile -Encoding UTF8NoBom -Raw - Write-Verbose -Verbose $buildInfoJsonContent - - $buildInfo = $buildInfoJsonContent | ConvertFrom-Json - $buildInfo.ReleaseDate = $dateTime - - $targetFile = "$ENV:PIPELINE_WORKSPACE/$fileName" - ConvertTo-Json -InputObject $buildInfo | Out-File $targetFile -Encoding ascii - - if ($stableRelease -or $fileName -eq "preview.json") { - Set-BuildVariable -Name CopyMainBuildInfo -Value YES - } else { - Set-BuildVariable -Name CopyMainBuildInfo -Value NO - } - - Set-BuildVariable -Name BuildInfoJsonFile -Value $targetFile - - ## Create 'lts.json' if it's the latest stable and also a LTS release. - - if ($fileName -eq "stable.json") { - if ($ltsRelease) { - $ltsFile = "$ENV:PIPELINE_WORKSPACE/lts.json" - Copy-Item -Path $targetFile -Destination $ltsFile -Force - Set-BuildVariable -Name LtsBuildInfoJsonFile -Value $ltsFile - Set-BuildVariable -Name CopyLTSBuildInfo -Value YES - } else { - Set-BuildVariable -Name CopyLTSBuildInfo -Value NO - } - - $releaseTag = $buildInfo.ReleaseTag - $version = $releaseTag -replace '^v' - $semVersion = [System.Management.Automation.SemanticVersion] $version - - $versionFile = "$ENV:PIPELINE_WORKSPACE/$($semVersion.Major)-$($semVersion.Minor).json" - Copy-Item -Path $targetFile -Destination $versionFile -Force - Set-BuildVariable -Name VersionBuildInfoJsonFile -Value $versionFile - Set-BuildVariable -Name CopyVersionBuildInfo -Value YES - } else { - Set-BuildVariable -Name CopyVersionBuildInfo -Value NO - } - displayName: Download and Capture NuPkgs - -- task: AzureFileCopy@4 - displayName: 'AzureBlob build info JSON file Copy' - inputs: - SourcePath: '$(BuildInfoJsonFile)' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: BuildInfo - condition: and(succeeded(), eq(variables['CopyMainBuildInfo'], 'YES')) - retryCountOnTaskFailure: 2 - -- task: AzureFileCopy@4 - displayName: 'AzureBlob build info ''lts.json'' Copy when needed' - inputs: - SourcePath: '$(LtsBuildInfoJsonFile)' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: BuildInfo - condition: and(succeeded(), eq(variables['CopyLTSBuildInfo'], 'YES')) - retryCountOnTaskFailure: 2 - -- task: AzureFileCopy@4 - displayName: 'AzureBlob build info ''Major-Minor.json'' Copy when needed' - inputs: - SourcePath: '$(VersionBuildInfoJsonFile)' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: BuildInfo - condition: and(succeeded(), eq(variables['CopyVersionBuildInfo'], 'YES')) - retryCountOnTaskFailure: 2 diff --git a/tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml b/tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml deleted file mode 100644 index 7c9306496ed..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml +++ /dev/null @@ -1,56 +0,0 @@ -parameters: -- name: sourceContainerName - type: string - default: 'source-container' - -- name: destinationContainerName - type: string - default: 'destination-container' - -- name: sourceStorageAccountName - type: string - default: 'source-storage-account' - -- name: destinationStorageAccountName - type: string - default: 'destination-storage-account' - -- name: blobPrefix - type: string - default: '$(Version)' - -steps: -- template: release-SetReleaseTagAndContainerName.yml - -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - Install-AzCopy - displayName: Install AzCopy - retryCountOnTaskFailure: 2 - -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - $azcopy = Find-AzCopy - Write-Verbose -Verbose "Found AzCopy: $azcopy" - - $sourceContainerName = "${{ parameters.sourceContainerName }}" - $destinationContainerName = "${{ parameters.destinationContainerName }}" - $sourceStorageAccountName = "${{ parameters.sourceStorageAccountName }}" - $destinationStorageAccountName = "${{ parameters.destinationStorageAccountName }}" - $blobPrefix = "${{ parameters.blobPrefix }}" - - $sourceBlobUrl = "https://${sourceStorageAccountName}.blob.core.windows.net/${sourceContainerName}/${blobPrefix}" - Write-Verbose -Verbose "Source blob url: $sourceBlobUrl" - $destinationBlobUrl = "https://${destinationStorageAccountName}.blob.core.windows.net/${destinationContainerName}" - Write-Verbose -Verbose "Destination blob url: $destinationBlobUrl" - - & $azcopy cp $sourceBlobUrl $destinationBlobUrl --recursive - - $packagesPath = Get-ChildItem -Path $(System.ArtifactsDirectory)\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName - Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" - Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - - displayName: Copy blobs - retryCountOnTaskFailure: 2 - env: - AZCOPY_AUTO_LOGIN_TYPE: MSI diff --git a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml deleted file mode 100644 index 64c4d1b6a24..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml +++ /dev/null @@ -1,110 +0,0 @@ -steps: -- checkout: self - clean: true - -- download: none - -- template: release-SetReleaseTagAndContainerName.yml - -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/PowerShell/build.psm1' - Install-AzCopy - displayName: Install AzCopy - retryCountOnTaskFailure: 2 - -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/PowerShell/build.psm1' - $azcopy = Find-AzCopy - Write-Verbose -Verbose "Found AzCopy: $azcopy" - - & $azcopy cp https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) $(System.ArtifactsDirectory) --recursive - - $packagesPath = Get-ChildItem -Path $(System.ArtifactsDirectory)\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName - Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" - Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - - displayName: Download Azure Artifacts - retryCountOnTaskFailure: 2 - env: - AZCOPY_AUTO_LOGIN_TYPE: MSI - -- pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty FullName - displayName: Capture downloaded artifacts - -- pwsh: | - git clone https://$(AzureDevOpsPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' - displayName: Clone Internal-Tools repository - -- pwsh: | - $Path = "$(PackagesRoot)" - $OutputPath = Join-Path $Path ‘hashes.sha256’ - $srcPaths = @($Path) - $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File - $checksums = $packages | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - $checksums | Out-File -FilePath $OutputPath -Force - $fileContent = Get-Content -Path $OutputPath -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent - displayName: Add sha256 hashes - -- checkout: ComplianceRepo - -- pwsh: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set release version' - -- template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(PackagesRoot)' - Build_Repository_Uri: 'https://github.com/powershell/powershell.git' - displayName: PowerShell Hashes SBOM - packageName: PowerShell Artifact Hashes - packageVersion: $(ReleaseVersion) - sourceScanPath: '$(PackagesRoot)' - -- pwsh: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } - - $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" - - if (-not (Test-Path $filePath)) { - throw "$filePath not found" - } - - $changelog = Get-Content -Path $filePath - - $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" - $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch - - $clContent = $changelog | ForEach-Object { - if ($_ -match $startPattern) { $outputLine = $true } - elseif ($_ -match $endPattern) { $outputLine = $false } - if ($outputLine) { $_} - } | Out-String - - Write-Verbose -Verbose "Selected content: `n$clContent" - - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) - displayName: Publish Release Draft diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml deleted file mode 100644 index 8591791de0e..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ /dev/null @@ -1,149 +0,0 @@ -parameters: - jobName: "" - displayName: "" - imageName: "" - globalToolExeName: 'pwsh.exe' - globalToolPackageName: 'PowerShell.Windows.x64' - - -jobs: -- job: ${{ parameters.jobName }} - displayName: ${{ parameters.displayName }} - pool: - # test - vmImage: ${{ parameters.imageName }} - variables: - - group: DotNetPrivateBuildAccess - - steps: - - checkout: self - clean: true - - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - patterns: '**/*.nupkg' - path: '$(Pipeline.Workspace)/releasePipeline/finalResults' - - - pwsh: | - $dotnetMetadataPath = "$(Build.SourcesDirectory)/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json).sdk.version - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - - Find-Dotnet - - if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) - { - $nugetFeed = ([xml](Get-Content $(Build.SourcesDirectory)/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value - if ($nugetFeed) { - Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet - Write-Verbose -Message "Register new package source 'dotnet'" -verbose - } - } - - ## Install latest version from the channel - - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion - Start-PSBootstrap - - Write-Verbose -Message "Installing .NET SDK completed." -Verbose - - displayName: Install .NET - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - $branch = $ENV:BUILD_SOURCEBRANCH - $version = $branch -replace '^.*(release[-/])v' - $vstsCommandString = "vso[task.setvariable variable=PowerShellVersion]$version" - Write-Verbose -Message "Version is $version" -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Set PowerShell Version - - - pwsh: | - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap - - $toolPath = New-Item -ItemType Directory "$(System.DefaultWorkingDirectory)/toolPath" | Select-Object -ExpandProperty FullName - - dotnet tool install --add-source "$ENV:PIPELINE_WORKSPACE/releasePipeline/finalResults" --tool-path $toolPath --version '$(PowerShellVersion)' '${{ parameters.globalToolPackageName }}' - - Get-ChildItem -Path $toolPath - - displayName: Install global tool - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - $toolPath = "$(System.DefaultWorkingDirectory)/toolPath/${{ parameters.globalToolExeName }}" - - if (-not (Test-Path $toolPath)) - { - throw "Tool is not installed at $toolPath" - } - else - { - Write-Verbose -Verbose "Tool found at: $toolPath" - } - displayName: Validate tool is installed - - - pwsh: | - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap - - $exeName = if ($IsWindows) { "pwsh.exe" } else { "pwsh" } - - $toolPath = "$(System.DefaultWorkingDirectory)/toolPath/${{ parameters.globalToolExeName }}" - - $source = (get-command -Type Application -Name dotnet | Select-Object -First 1 -ExpandProperty source) - $target = (Get-ChildItem $source).target - - # If we find a symbolic link for dotnet, then we need to split the filename off the target. - if ($target) { - Write-Verbose -Verbose "Splitting target: $target" - $target = Split-Path $target - } - - Write-Verbose -Verbose "target is set as $target" - - $env:DOTNET_ROOT = (resolve-path -Path (Join-Path (split-path $source) $target)).ProviderPath - - Write-Verbose -Verbose "DOTNET_ROOT: $env:DOTNET_ROOT" - Get-ChildItem $env:DOTNET_ROOT - - $versionFound = & $toolPath -c '$PSVersionTable.PSVersion.ToString()' - - if ( '$(PowerShellVersion)' -ne $versionFound) - { - throw "Expected version of global tool not found. Installed version is $versionFound" - } - else - { - write-verbose -verbose "Found expected version: $versionFound" - } - - $dateYear = & $toolPath -c '(Get-Date).Year' - - if ( $dateYear -ne [DateTime]::Now.Year) - { - throw "Get-Date returned incorrect year: $dateYear" - } - else - { - write-verbose -verbose "Got expected year: $dateYear" - } - displayName: Basic validation - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) diff --git a/tools/releaseBuild/azureDevOps/templates/release-MakeContainerPublic.yml b/tools/releaseBuild/azureDevOps/templates/release-MakeContainerPublic.yml deleted file mode 100644 index 65d5ea50191..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-MakeContainerPublic.yml +++ /dev/null @@ -1,20 +0,0 @@ -steps: -- download: none - -- template: release-SetReleaseTagAndContainerName.yml - -- pwsh: | - az login --service-principal -u $(az_url) -p $(az_key) --tenant $(az_name) - displayName: az login - -- pwsh: | - az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion) --public-access blob - displayName: Make container public - -- pwsh: | - az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion)-gc --public-access blob - displayName: Make guest configuration miminal package container public - -- pwsh: | - az logout - displayName: az logout diff --git a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml deleted file mode 100644 index a9591b2d251..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml +++ /dev/null @@ -1,81 +0,0 @@ -jobs: -- job: CreateMSIXBundle - displayName: Create .msixbundle file - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: msixTools - - group: 'Azure Blob variable group' - - steps: - - template: release-SetReleaseTagAndContainerName.yml - - - task: DownloadPipelineArtifact@2 - retryCountOnTaskFailure: 2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - patterns: '**/*.msix' - path: '$(Pipeline.Workspace)\releasePipeline\msix' - - - pwsh: | - $cmd = Get-Command makeappx.exe -ErrorAction Ignore - if ($cmd) { - Write-Verbose -Verbose 'makeappx available in PATH' - $exePath = $cmd.Source - } else { - $toolsDir = '$(Pipeline.Workspace)\releasePipeline\tools' - New-Item $toolsDir -Type Directory -Force > $null - Invoke-RestMethod -Uri '$(makeappUrl)' -OutFile "$toolsDir\makeappx.zip" - Expand-Archive "$toolsDir\makeappx.zip" -DestinationPath "$toolsDir\makeappx" -Force - $exePath = "$toolsDir\makeappx\makeappx.exe" - - Write-Verbose -Verbose 'makeappx was installed:' - Get-ChildItem -Path $toolsDir -Recurse - } - - $vstsCommandString = "vso[task.setvariable variable=MakeAppxPath]$exePath" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Install makeappx tool - retryCountOnTaskFailure: 1 - - - pwsh: | - $sourceDir = '$(Pipeline.Workspace)\releasePipeline\msix' - $file = Get-ChildItem $sourceDir | Select-Object -First 1 - $prefix = ($file.BaseName -split "-win")[0] - $pkgName = "$prefix.msixbundle" - Write-Verbose -Verbose "Creating $pkgName" - - $makeappx = '$(MakeAppxPath)' - $outputDir = "$sourceDir\output" - New-Item $outputDir -Type Directory -Force > $null - & $makeappx bundle /d $sourceDir /p "$outputDir\$pkgName" - - Get-ChildItem -Path $sourceDir -Recurse - $vstsCommandString = "vso[task.setvariable variable=BundleDir]$outputDir" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Create MsixBundle - retryCountOnTaskFailure: 1 - - - task: AzureFileCopy@4 - displayName: 'Upload MSIX Bundle package to Az Blob' - retryCountOnTaskFailure: 2 - inputs: - SourcePath: '$(BundleDir)/*.msixbundle' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)-private' - resourceGroup: '$(StorageResourceGroup)' - condition: succeeded() diff --git a/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml b/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml deleted file mode 100644 index 861cf48c35a..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml +++ /dev/null @@ -1,57 +0,0 @@ -parameters: - - name: skipPublish - default: false - type: boolean - -steps: -- template: release-SetReleaseTagAndContainerName.yml - -- pwsh: | - $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Set Package version - -- pwsh: | - $branch = 'main-mirror' - $gitArgs = "clone", - "--verbose", - "--branch", - "$branch", - "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", - '$(Pipeline.Workspace)/tools' - $gitArgs | Write-Verbose -Verbose - git $gitArgs - displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - -- task: PipAuthenticate@1 - inputs: - artifactFeeds: 'pmc' - pythonDownloadServiceConnections: pmcDownload - -- pwsh: | - pip install pmc-cli - - $newPath = (resolve-path '~/.local/bin').providerpath - $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Install pmc cli - -- pwsh: | - $metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json - $params = @{ - ReleaseTag = "$(ReleaseTag)" - AadClientId = "$(PmcCliClientID)" - BlobFolderName = "$(AzureVersion)" - LTS = $metadata.LTSRelease.Latest - ForProduction = $true - SkipPublish = $${{ parameters.skipPublish }} - MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' - } - - $params | Out-String -width 9999 -Stream | write-Verbose -Verbose - - & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params - displayName: Run release script diff --git a/tools/releaseBuild/azureDevOps/templates/release-PublishSymbols.yml b/tools/releaseBuild/azureDevOps/templates/release-PublishSymbols.yml deleted file mode 100644 index db2cc86e259..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-PublishSymbols.yml +++ /dev/null @@ -1,51 +0,0 @@ -steps: -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: results - path: '$(Pipeline.Workspace)\results' - itemPattern: | - **/* - !**/*signed.zip - -- pwsh: | - Write-Verbose -Verbose "Enumerating $(Pipeline.Workspace)\results" - $downloadedArtifacts = Get-ChildItem -Recurse "$(Pipeline.Workspace)\results" - $downloadedArtifacts - $expandedRoot = New-Item -Path "$(Pipeline.Workspace)/expanded" -ItemType Directory -Verbose - $symbolsRoot = New-Item -Path "$(Pipeline.Workspace)/symbols" -ItemType Directory -Verbose - - $downloadedArtifacts | ForEach-Object { - $destFolder = New-Item -Path "$expandedRoot/$($_.BaseName)/" -ItemType Directory -Verbose - Expand-Archive -Path $_.FullName -DestinationPath $destFolder -Force - - $symbolsZipFile = Join-Path -Path $destFolder -ChildPath "symbols.zip" - $symbolZipFileContents = New-Item -Path "$destFolder/Symbols-$($_.BaseName)" -ItemType Directory -Verbose - Expand-Archive -Path $symbolsZipFile -DestinationPath $symbolZipFileContents -Force - - $symbolsToPublish = New-Item -Path "$symbolsRoot/$($_.BaseName)" -ItemType Directory -Verbose - - Get-ChildItem -Path $symbolZipFileContents -Recurse -Filter '*.pdb' | ForEach-Object { - Copy-Item -Path $_.FullName -Destination $symbolsToPublish -Verbose - } - } - - Write-Verbose -Verbose "Enumerating $symbolsRoot" - Get-ChildItem -Path $symbolsRoot -Recurse - $vstsCommandString = "vso[task.setvariable variable=SymbolsPath]$symbolsRoot" - Write-Verbose -Message "$vstsCommandString" -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Expand and capture symbols folders -- task: PublishSymbols@2 - inputs: - symbolsFolder: '$(SymbolsPath)' - searchPattern: '**/*.pdb' - indexSources: false - publishSymbols: true - symbolServerType: teamServices - detailedLog: true diff --git a/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml b/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml deleted file mode 100644 index 33a72f56bbb..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml +++ /dev/null @@ -1,56 +0,0 @@ -parameters: - - name: skipPublish - default: false - type: boolean - -steps: -- task: DownloadPipelineArtifact@2 - condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - patterns: '**/*.nupkg' - path: '$(Pipeline.Workspace)/releasePipeline/finalResults' - -- task: DownloadPipelineArtifact@2 - condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - -- pwsh: | - #Exclude all global tool packages. Their names start with 'PowerShell.' - $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" - Copy-Item "$ENV:PIPELINE_WORKSPACE/releasePipeline/finalResults/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose - - $releaseVersion = Get-Content "$ENV:PIPELINE_WORKSPACE/releasePipeline/metadata/release.json" | ConvertFrom-Json | Select-Object -ExpandProperty 'ReleaseVersion' - $globalToolPath = "$ENV:PIPELINE_WORKSPACE/releasePipeline/finalResults/PowerShell.$releaseVersion.nupkg" - - if ($releaseVersion -notlike '*-*') { - # Copy the global tool package for stable releases - Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" - } - - Get-ChildItem "$(Pipeline.Workspace)/release" -recurse - displayName: Download and capture nupkgs - condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) - -- task: NuGetCommand@2 - displayName: 'NuGet push' - condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) - inputs: - command: push - packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' - nuGetFeedType: external - publishFeedCredentials: PowerShellNuGetOrgPush diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml deleted file mode 100644 index 93fb0bf07cb..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ /dev/null @@ -1,148 +0,0 @@ -parameters: - jobName: "" - displayName: "" - imageName: "" - -jobs: -- job: ${{ parameters.jobName }} - displayName: ${{ parameters.displayName }} - pool: - # testing - vmImage: ${{ parameters.imageName }} - variables: - - group: mscodehub-feed-read-general - - group: mscodehub-feed-read-akv - - group: DotNetPrivateBuildAccess - steps: - - checkout: self - clean: true - - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - patterns: '**/*.nupkg' - path: '$(Pipeline.Workspace)/releasePipeline/finalResults' - - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: metadata - path: '$(Pipeline.Workspace)/releasePipeline/metadata' - - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self - parameters: - repoRoot: $(Build.SourcesDirectory) - - - pwsh: | - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - - Write-Verbose -Verbose "Capture hosting folder files" - Get-ChildItem '$(Build.SourcesDirectory)/test/hosting' - - # The above cmdlet creates a lower-case nuget.config. There also exists a NuGet.config which we needed to replace. - # Hence the following workaround - - if (-not $IsWindows) { - Move-Item -Path '$(Build.SourcesDirectory)/test/hosting/nuget.config' -Destination '$(Build.SourcesDirectory)/test/hosting/NuGet.Config' -Force -ErrorAction Continue - Write-Verbose -Verbose "Capture hosting folder files after Move-Item" - Get-ChildItem '$(Build.SourcesDirectory)/test/hosting' - } - - if(-not (Test-Path "$(Build.SourcesDirectory)/test/hosting/NuGet.Config")) - { - throw "NuGet.Config is not created" - } - else - { - Write-Verbose -Verbose "Capture NuGet.Config contents" - Get-Content "$(Build.SourcesDirectory)/test/hosting/NuGet.Config" -Raw - } - displayName: Insert internal nuget feed - - - pwsh: | - $dotnetMetadataPath = "$(Build.SourcesDirectory)/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json).sdk.version - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - - Find-Dotnet - - if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) - { - $nugetFeed = ([xml](Get-Content $(Build.SourcesDirectory)/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value - - if ($nugetFeed) { - Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet - Write-Verbose -Message "Register new package source 'dotnet'" -verbose - } - } - - ## Install latest version from the channel - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion - - Start-PSBootstrap - - Write-Verbose -Message "Installing .NET SDK completed." -Verbose - - displayName: Install .NET - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap - - $localLocation = "$(Pipeline.Workspace)/releasePipeline/finalResults" - $xmlElement = @" - - - - "@ - - $releaseVersion = Get-Content "$(Pipeline.Workspace)/releasePipeline/metadata/release.json" | ConvertFrom-Json | Select-Object -ExpandProperty 'ReleaseVersion' - - Set-Location -Path $(Build.SourcesDirectory)/test/hosting - - Get-ChildItem - - ## register the packages download directory in the nuget file - $nugetConfigContent = Get-Content ./NuGet.Config -Raw - $updateNugetContent = $nugetConfigContent.Replace("", $xmlElement) - - $updateNugetContent | Out-File ./NuGet.Config -Encoding ascii - - Get-Content ./NuGet.Config - - # Add workaround to unblock xUnit testing see issue: https://github.com/dotnet/sdk/issues/26462 - $dotnetPath = if ($IsWindows) { "$env:LocalAppData\Microsoft\dotnet" } else { "$env:HOME/.dotnet" } - $env:DOTNET_ROOT = $dotnetPath - - dotnet --info - dotnet restore - dotnet test /property:RELEASE_VERSION=$releaseVersion --test-adapter-path:. "--logger:xunit;LogFilePath=$(System.DefaultWorkingDirectory)/test-hosting.xml" - - displayName: Restore and execute tests - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - task: PublishTestResults@2 - displayName: 'Publish Test Results **\test-hosting.xml' - inputs: - testResultsFormat: XUnit - testResultsFiles: '**\test-hosting.xml' diff --git a/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml b/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml deleted file mode 100644 index 7e88624b45c..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml +++ /dev/null @@ -1,26 +0,0 @@ -steps: -- pwsh: | - $variable = 'releaseTag' - $branch = $ENV:BUILD_SOURCEBRANCH - if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') - { - throw "Branch name is not in release format: '$branch'" - } - - $releaseTag = $Branch -replace '^.*((release|rebuild)/)' - $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" - Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Set Release Tag - -- pwsh: | - $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Set container name diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml deleted file mode 100644 index fa42064602e..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ /dev/null @@ -1,71 +0,0 @@ -jobs: -- job: UpdateDepsFiles - displayName: Update deps files - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - variables: - - group: 'Azure Blob variable group' - steps: - - checkout: self - clean: true - - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - patterns: '**/PowerShell*-win-x64.zip' - path: '$(Pipeline.Workspace)/releasePipeline/finalResults' - - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: BuildInfoJson - path: '$(Pipeline.Workspace)/releasePipeline/BuildInfoJson' - - - pwsh: | - $fileName = (Get-Item "$ENV:PIPELINE_WORKSPACE/releasePipeline/BuildInfoJson/*.json").BaseName - if ($fileName -notin 'stable','preview') - { - throw "Unexpected fileName: $fileName" - } - - $vstsCommand = "vso[task.setvariable variable=BlobPrefix]$fileName" - Write-Verbose -Verbose $vstsCommand - Write-Host "##$vstsCommand" - displayName: Determine container name - - - pwsh: | - $zipFile = Get-Item "$ENV:PIPELINE_WORKSPACE/releasePipeline/finalResults/PowerShell*-win-x64.zip" -Exclude *-symbols-* - Write-Verbose -Verbose "zipFile: $zipFile" - Expand-Archive -Path $zipFile -Destination "$ENV:PIPELINE_WORKSPACE/expanded" - - $pwshDepsFile = Get-Item "$ENV:PIPELINE_WORKSPACE/expanded/pwsh.deps.json" - $vstsCommand = "vso[task.setvariable variable=FileToUpload]$pwshDepsFile" - Write-Verbose -Verbose $vstsCommand - Write-Host "##$vstsCommand" - displayName: Determine file to upload - - - task: AzureFileCopy@4 - displayName: 'AzureBlob pwsh.deps.json file Copy' - inputs: - SourcePath: '$(FileToUpload)' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: ps-deps-json - blobPrefix: '$(BlobPrefix)' - retryCountOnTaskFailure: 2 diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml deleted file mode 100644 index 7f2c816a20f..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml +++ /dev/null @@ -1,92 +0,0 @@ -parameters: - jobName: "" - displayName: "" - imageName: "" - packageNamePattern: "" - use1ES: false - -jobs: -- job: ${{ parameters.jobName }} - displayName: ${{ parameters.displayName }} - variables: - - group: DotNetPrivateBuildAccess - pool: - ${{ if eq(parameters.use1ES, 'false') }}: - vmImage: ${{ parameters.imageName }} - ${{ else }}: - name: 'PS-MSCodeHub-ARM' # add ImageOverride to select image - steps: - - checkout: self - clean: true - - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - patterns: '${{ parameters.packageNamePattern }}' - path: '$(Pipeline.Workspace)/releasePipeline/finalResults' - - - pwsh: | - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap - Write-Verbose -Message "Installing .NET SDK completed." -Verbose - displayName: Install .NET - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - Get-ChildItem -Path '$(Pipeline.Workspace)/releasePipeline/finalResults' -Recurse - displayName: Capture downloaded package - - - pwsh: | - $destPath = New-Item '$(Pipeline.Workspace)/releasePipeline/finalResults/fxd' -ItemType Directory - $packageNameFilter = '${{ parameters.packageNamePattern }}' - - if ($packageNameFilter.EndsWith('tar.gz')) { - $package = @(Get-ChildItem -Path '$(Pipeline.Workspace)/releasePipeline/finalResults/*.tar.gz') - Write-Verbose -Verbose "Package: $package" - if ($package.Count -ne 1) { - throw 'Only 1 package was expected.' - } - tar -xvf $package.FullName -C $destPath - } - else { - $package = @(Get-ChildItem -Path '$(Pipeline.Workspace)/releasePipeline/finalResults/*.zip') - Write-Verbose -Verbose "Package: $package" - if ($package.Count -ne 1) { - throw 'Only 1 package was expected.' - } - Expand-Archive -Path $package.FullName -Destination "$destPath" -Verbose - } - displayName: Expand fxd package - - - pwsh: | - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Find-Dotnet -SetDotnetRoot - Write-Verbose -Verbose "DOTNET_ROOT: $env:DOTNET_ROOT" - Write-Verbose -Verbose "Check dotnet install" - dotnet --info - Write-Verbose -Verbose "Start test" - $packageNameFilter = '${{ parameters.packageNamePattern }}' - $pwshExeName = if ($packageNameFilter.EndsWith('tar.gz')) { 'pwsh' } else { 'pwsh.exe' } - $pwshPath = Join-Path '$(Pipeline.Workspace)/releasePipeline/finalResults/fxd' $pwshExeName - - if ($IsLinux) { - chmod u+x $pwshPath - } - - $pwshDllPath = Join-Path '$(Pipeline.Workspace)/releasePipeline/finalResults/fxd' 'pwsh.dll' - - $actualOutput = & dotnet $pwshDllPath -c 'Start-ThreadJob -ScriptBlock { "1" } | Wait-Job | Receive-Job' - Write-Verbose -Verbose "Actual output: $actualOutput" - if ($actualOutput -ne 1) { - throw "Actual output is not as expected" - } - displayName: Test package diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml deleted file mode 100644 index 3fd560cbd00..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml +++ /dev/null @@ -1,49 +0,0 @@ -steps: -- checkout: self - clean: true - -- pwsh: | - Get-ChildItem ENV: - displayName: Capture environment - -- template: release-SetReleaseTagAndContainerName.yml - -- pwsh: | - $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks - Write-Host $name - Write-Host "##vso[build.updatebuildnumber]$name" - displayName: Set Release Name - -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: PowerShellCore - pipeline: '696' - preferTriggeringPipeline: true - runVersion: latestFromBranch - runBranch: '$(Build.SourceBranch)' - artifact: finalResults - path: $(System.ArtifactsDirectory) - - -- pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name - displayName: Capture Artifact Listing - -- pwsh: | - Install-module Pester -Scope CurrentUser -Force -MaximumVersion 4.99 - displayName: Install Pester - condition: succeededOrFailed() - -- pwsh: | - Import-module './build.psm1' - Import-module './tools/packaging' - $env:PACKAGE_FOLDER = '$(System.ArtifactsDirectory)' - $path = Join-Path -Path $pwd -ChildPath './packageReleaseTests.xml' - $results = invoke-pester -Script './tools/packaging/releaseTests' -OutputFile $path -OutputFormat NUnitXml -PassThru - Write-Host "##vso[results.publish type=NUnit;mergeResults=true;runTitle=Package Release Tests;publishRunAttachments=true;resultFiles=$path;]" - if($results.TotalCount -eq 0 -or $results.FailedCount -gt 0) - { - throw "Package Release Tests failed" - } - displayName: Run packaging release tests diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml deleted file mode 100644 index 8e41fbc4a55..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml +++ /dev/null @@ -1,93 +0,0 @@ -steps: -- pwsh: | - Get-ChildItem ENV: - displayName: Capture environment - -- template: release-SetReleaseTagAndContainerName.yml - -- pwsh: | - $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks - Write-Host $name - Write-Host "##vso[build.updatebuildnumber]$name" - displayName: Set Release Name - -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - $azcopy = Find-AzCopy - Write-Verbose -Verbose "Found AzCopy: $azcopy" - - & $azcopy cp https://$(StorageAccount).blob.core.windows.net/$(AzureVersion)/* $(System.ArtifactsDirectory) --recursive - - displayName: Download Azure Artifacts - env: - AZCOPY_AUTO_LOGIN_TYPE: MSI - -- pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name - displayName: Capture Artifact Listing - -- pwsh: | - $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.rpm | ForEach-Object { - if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d+\.\d+\.\d+(_[a-z]*\.\d+)?-1.(rh|cm).(x86_64|aarch64)\.rpm') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } - } - if($message.count -gt 0){throw ($message | out-string)} - displayName: Validate RPM package names - -- pwsh: | - $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.tar.gz | ForEach-Object { - if($_.Name -notmatch 'powershell-(lts-)?\d+\.\d+\.\d+\-([a-z]*.\d+\-)?(linux|osx|linux-musl)+\-(x64\-fxdependent|x64|arm32|arm64|x64\-musl-noopt\-fxdependent)\.(tar\.gz)') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } - } - if($message.count -gt 0){throw ($message | out-string)} - displayName: Validate Tar.Gz Package Names - -- pwsh: | - $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.pkg | ForEach-Object { - if($_.Name -notmatch 'powershell-(lts-)?\d+\.\d+\.\d+\-([a-z]*.\d+\-)?osx(\.10\.12)?\-(x64|arm64)\.pkg') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } - } - if($message.count -gt 0){throw ($message | out-string)} - displayName: Validate PKG Package Names - -- pwsh: | - $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -include *.zip, *.msi | ForEach-Object { - if($_.Name -notmatch 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(fxdependent|x64|arm64|x86|fxdependentWinDesktop)\.(msi|zip){1}') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } - } - - if($message.count -gt 0){throw ($message | out-string)} - displayName: Validate Zip and MSI Package Names - -- pwsh: | - $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.deb | ForEach-Object { - if($_.Name -notmatch 'powershell(-preview|-lts)?_\d+\.\d+\.\d+([\-~][a-z]*.\d+)?-\d\.deb_amd64\.deb') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } - } - if($message.count -gt 0){throw ($message | out-string)} - displayName: Validate Deb Package Names diff --git a/tools/releaseBuild/azureDevOps/templates/release/approvalJob.yml b/tools/releaseBuild/azureDevOps/templates/release/approvalJob.yml deleted file mode 100644 index b34cc4c75b6..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/release/approvalJob.yml +++ /dev/null @@ -1,35 +0,0 @@ -parameters: - - name: displayName - type: string - - name: instructions - type: string - - name: jobName - type: string - default: approval - - name: timeoutInMinutes - type: number - # 2 days - default: 2880 - - name: onTimeout - type: string - default: 'reject' - values: - - resume - - reject - - name: dependsOnJob - type: string - default: '' - -jobs: - - job: ${{ parameters.jobName }} - dependsOn: ${{ parameters.dependsOnJob }} - displayName: ${{ parameters.displayName }} - pool: server - timeoutInMinutes: 4320 # job times out in 3 days - steps: - - task: ManualValidation@0 - displayName: ${{ parameters.displayName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - inputs: - instructions: ${{ parameters.instructions }} - onTimeout: ${{ parameters.onTimeout }} diff --git a/tools/releaseBuild/azureDevOps/templates/shouldSign.yml b/tools/releaseBuild/azureDevOps/templates/shouldSign.yml deleted file mode 100644 index e3c38cb29d5..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/shouldSign.yml +++ /dev/null @@ -1,29 +0,0 @@ -steps: -- powershell: | - $shouldSign = $true - $authenticodeCert = 'CP-230012' - $msixCert = 'CP-230012' - - if($env:IS_DAILY -eq 'true') - { - $authenticodeCert = 'CP-460906' - } - - if($env:SKIP_SIGNING -eq 'Yes') - { - $shouldSign = $false - } - - $vstsCommandString = "vso[task.setvariable variable=SHOULD_SIGN]$($shouldSign.ToString().ToLowerInvariant())" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $vstsCommandString = "vso[task.setvariable variable=MSIX_CERT]$($msixCert)" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $vstsCommandString = "vso[task.setvariable variable=AUTHENTICODE_CERT]$($authenticodeCert)" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - displayName: 'Set SHOULD_SIGN Variable' diff --git a/tools/releaseBuild/azureDevOps/templates/sign-build-file.yml b/tools/releaseBuild/azureDevOps/templates/sign-build-file.yml deleted file mode 100644 index a584e15e27c..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/sign-build-file.yml +++ /dev/null @@ -1,328 +0,0 @@ -steps: -- pwsh: | - $platform = '$(runtime)' -match '^linux' ? 'linux' : 'windows' - $vstsCommandString = "vso[task.setvariable variable=ArtifactPlatform]$platform" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Set artifact platform - -- task: DownloadPipelineArtifact@2 - inputs: - artifactName: '$(unsignedBuildArtifactContainer)' - itemPattern: '$(unsignedBuildArtifactName)' - -- pwsh: | - Get-ChildItem "$(Pipeline.Workspace)\*" -Recurse - displayName: 'Capture Downloaded Artifacts' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - -- checkout: self - clean: true - path: $(repoFolder) - -- template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - -- template: cloneToOfficialPath.yml - -- pwsh: | - $zipFileFilter = '$(unsignedBuildArtifactName)' - $zipFileFilter = $zipFileFilter.Replace('**/', '') - - Write-Verbose -Verbose -Message "zipFileFilter = $zipFileFilter" - - Write-Verbose -Verbose -Message "Looking for $(Pipeline.Workspace)\$(unsignedBuildArtifactName)" - - $zipFilePath = Get-ChildItem -Path '$(Pipeline.Workspace)\$(unsignedBuildArtifactName)' -recurse - - if (-not (Test-Path $zipFilePath)) - { - throw "zip file not found: $zipfilePath" - } - - if ($zipFilePath.Count -ne 1) { - Write-Verbose "zip filename" -verbose - $zipFilePath | Out-String | Write-Verbose -Verbose - throw 'multiple zip files found when 1 was expected' - } - - $expandedFolderName = [System.io.path]::GetFileNameWithoutExtension($zipfilePath) - $expandedFolderPath = Join-Path '$(Pipeline.Workspace)' 'expanded' $expandedFolderName - - Write-Verbose -Verbose -Message "Expaning $zipFilePath to $expandedFolderPath" - - New-Item -Path $expandedFolderPath -ItemType Directory - Expand-Archive -Path $zipFilePath -DestinationPath $expandedFolderPath - - if (-not (Test-Path $expandedFolderPath\pwsh.exe) ) { - throw 'zip did not expand as expected' - } - else { - $vstsCommandString = "vso[task.setvariable variable=BinPath]$expandedFolderPath" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - } - - displayName: Expand zip packages - condition: eq(variables['ArtifactPlatform'], 'windows') - -- pwsh: | - $tarPackageName = '$(unsignedBuildArtifactName)' - - Write-Verbose -Verbose -Message "tarPackageName = $tarPackageName" - - $tarPackagePath = Join-Path '$(Pipeline.Workspace)' $tarPackageName - - Write-Verbose -Verbose -Message "Looking for: $tarPackagePath" - - $expandedPathFolderName = $tarPackageName -replace '.tar.gz', '' - $expandedFolderPath = Join-Path '$(Pipeline.Workspace)' 'expanded' $expandedPathFolderName - - if (-not (Test-Path $tarPackagePath)) - { - throw "tar file not found: $tarPackagePath" - } - - Write-Verbose -Verbose -Message "Expanding $tarPackagePath to $expandedFolderPath" - - New-Item -Path $expandedFolderPath -ItemType Directory - tar -xf $tarPackagePath -C $expandedFolderPath - - if (-not (Test-Path $expandedFolderPath/pwsh) ) { - throw 'tar.gz did not expand as expected' - } - else { - $vstsCommandString = "vso[task.setvariable variable=BinPath]$expandedFolderPath" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - } - - Write-Verbose -Verbose "File permisions after expanding" - Get-ChildItem -Path "$expandedFolderPath/pwsh" | Select-Object -Property 'unixmode', 'size', 'name' - displayName: Expand tar.gz packages - condition: eq(variables['ArtifactPlatform'], 'linux') - -- template: insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(PowerShellRoot) - -- pwsh: | - Set-Location $env:POWERSHELLROOT - import-module "$env:POWERSHELLROOT/build.psm1" - Sync-PSTags -AddRemoteIfMissing - displayName: SyncTags - condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - -- checkout: ComplianceRepo - clean: true - path: $(complianceRepoFolder) - -- template: shouldSign.yml - -- pwsh: | - $fullSymbolsFolder = '$(BinPath)' - Write-Verbose -Verbose "fullSymbolsFolder == $fullSymbolsFolder" - - Get-ChildItem -Recurse $fullSymbolsFolder | out-string | Write-Verbose -Verbose - - $filesToSignDirectory = "$(System.ArtifactsDirectory)\toBeSigned" - - if ((Test-Path -Path $filesToSignDirectory)) { - Remove-Item -Path $filesToSignDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force - - $signedFilesDirectory = "$(System.ArtifactsDirectory)\signed" - - if ((Test-Path -Path $signedFilesDirectory)) { - Remove-Item -Path $signedFilesDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force - - $itemsToCopyWithRecurse = @( - "$($fullSymbolsFolder)\*.ps1" - "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" - ) - - $itemsToCopy = @{ - "$($fullSymbolsFolder)\*.ps1" = "" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Host\Microsoft.PowerShell.Host.psd1" = "Modules\Microsoft.PowerShell.Host" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Management\Microsoft.PowerShell.Management.psd1" = "Modules\Microsoft.PowerShell.Management" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Microsoft.PowerShell.Security.psd1" = "Modules\Microsoft.PowerShell.Security" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Utility\Microsoft.PowerShell.Utility.psd1" = "Modules\Microsoft.PowerShell.Utility" - "$($fullSymbolsFolder)\pwsh.dll" = "" - "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" - } - - ## Windows only modules - - if('$(ArtifactPlatform)' -eq 'windows') { - $itemsToCopy += @{ - "$($fullSymbolsFolder)\pwsh.exe" = "" - "$($fullSymbolsFolder)\Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" - "$($fullSymbolsFolder)\Microsoft.WSMan.*.dll" = "" - "$($fullSymbolsFolder)\Modules\CimCmdlets\CimCmdlets.psd1" = "Modules\CimCmdlets" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Security.types.ps1xml" = "Modules\Microsoft.PowerShell.Security" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Microsoft.PowerShell.Diagnostics.psd1" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" - "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" - "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" - } - } - else { - $itemsToCopy += @{ - "$($fullSymbolsFolder)\pwsh" = "" - } - } - - $itemsToExclude = @( - # This package is retrieved from https://www.github.com/powershell/MarkdownRender - "$($fullSymbolsFolder)\Microsoft.PowerShell.MarkdownRender.dll" - ) - - Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" - Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude - - foreach($pattern in $itemsToCopy.Keys) { - $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern - $null = New-Item -ItemType Directory -Path $destinationFolder -Force - Write-Verbose -verbose "copying $pattern to $destinationFolder" - Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose - } - displayName: 'Prepare files to be signed' - -- template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\toBeSigned - signOutputPath: $(System.ArtifactsDirectory)\signed - certificateId: "$(AUTHENTICODE_CERT)" - pattern: | - **\*.dll - **\*.psd1 - **\*.psm1 - **\*.ps1xml - **\*.ps1 - **\*.exe - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Authenticode sign our binaries - -- pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' - $BuildPath = '$(BinPath)' - Write-Verbose -Verbose -Message "BuildPath: $BuildPath" - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - $dlls = Get-ChildItem $BuildPath\*.dll, $BuildPath\*.exe -Recurse - $signatures = $dlls | Get-AuthenticodeSignature - $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch '^CN=Microsoft.*'}| select-object -ExpandProperty Path - - Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" - - $filesToSignDirectory = "$(System.ArtifactsDirectory)\thirdPartyToBeSigned" - if (Test-Path $filesToSignDirectory) { - Remove-Item -Path $filesToSignDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force -Verbose - - $signedFilesDirectory = "$(System.ArtifactsDirectory)\thirdPartySigned" - if (Test-Path $signedFilesDirectory) { - Remove-Item -Path $signedFilesDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force -Verbose - - $missingSignatures | ForEach-Object { - $pathWithoutLeaf = Split-Path $_ - $relativePath = $pathWithoutLeaf.replace($BuildPath,'') - Write-Verbose -Verbose -Message "relativePath: $relativePath" - $targetDirectory = Join-Path -Path $filesToSignDirectory -ChildPath $relativePath - Write-Verbose -Verbose -Message "targetDirectory: $targetDirectory" - if(!(Test-Path $targetDirectory)) - { - $null = New-Item -ItemType Directory -Path $targetDirectory -Force -Verbose - } - Copy-Item -Path $_ -Destination $targetDirectory - } - - displayName: Create ThirdParty Signing Folder - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - -- template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\thirdPartyToBeSigned - signOutputPath: $(System.ArtifactsDirectory)\thirdPartySigned - certificateId: "CP-231522" - pattern: | - **\*.dll - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign ThirdParty binaries - -- pwsh: | - Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' - displayName: Capture ThirdParty Signed files - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - -- pwsh: | - Import-Module '$(PowerShellRoot)/build.psm1' -Force - Import-Module '$(PowerShellRoot)/tools/packaging' -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' - $BuildPath = '$(BinPath)' - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - if ($env:BuildConfiguration -eq 'minSize') { - ## Remove XML files when making a min-size package. - Remove-Item "$BuildPath/*.xml" -Force - } - displayName: Merge ThirdParty signed files with Build - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - -- pwsh: | - $uploadFolder = '$(BinPath)' - $containerName = '$(signedArtifactContainer)' - - Write-Verbose -Verbose "File permissions after signing" - Get-ChildItem $uploadFolder\pwsh | Select-Object -Property 'unixmode', 'size', 'name' - - $uploadTarFilePath = Join-Path '$(System.ArtifactsDirectory)' '$(signedBuildArtifactName)' - Write-Verbose -Verbose -Message "Creating tar.gz - $uploadTarFilePath" - tar -czvf $uploadTarFilePath -C $uploadFolder * - - Get-ChildItem '$(System.ArtifactsDirectory)' | Out-String | Write-Verbose -Verbose - - Write-Host "##vso[artifact.upload containerfolder=$containerName;artifactname=$containerName]$uploadTarFilePath" - displayName: Upload signed tar.gz files to artifacts - condition: eq(variables['ArtifactPlatform'], 'linux') - retryCountOnTaskFailure: 2 - - -- pwsh: | - $uploadFolder = '$(BinPath)' - $containerName = '$(signedArtifactContainer)' - - Get-ChildItem $uploadFolder -Recurse | Out-String | Write-Verbose -Verbose - - $uploadZipFilePath = Join-Path '$(System.ArtifactsDirectory)' 'PowerShell-$(Version)$(signedBuildArtifactName)' - Write-Verbose -Verbose -Message "Creating zip - $uploadZipFilePath" - Compress-Archive -Path $uploadFolder/* -DestinationPath $uploadZipFilePath -Verbose - - Get-ChildItem '$(System.ArtifactsDirectory)' | Out-String | Write-Verbose -Verbose - - Write-Host "##vso[artifact.upload containerfolder=$containerName;artifactname=$containerName]$uploadZipFilePath" - displayName: Upload signed zip files to artifacts - condition: eq(variables['ArtifactPlatform'], 'windows') - retryCountOnTaskFailure: 2 - - -- template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/signBuildFiles.yml b/tools/releaseBuild/azureDevOps/templates/signBuildFiles.yml deleted file mode 100644 index a7c7c640ce7..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/signBuildFiles.yml +++ /dev/null @@ -1,189 +0,0 @@ -parameters: - binLocation: '' - buildPrefixName: '' - addWindowsModules: 'false' - -steps: -- pwsh: | - $fullSymbolsFolder = Join-Path $(System.ArtifactsDirectory) "${{ parameters.binLocation }}" - - Write-Verbose -Verbose "fullSymbolsFolder == $fullSymbolsFolder" - - Get-ChildItem -Recurse $fullSymbolsFolder | out-string | Write-Verbose -Verbose - - $filesToSignDirectory = "$(System.ArtifactsDirectory)\toBeSigned" - - if ((Test-Path -Path $filesToSignDirectory)) { - Remove-Item -Path $filesToSignDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force - - $signedFilesDirectory = "$(System.ArtifactsDirectory)\signed" - - if ((Test-Path -Path $signedFilesDirectory)) { - Remove-Item -Path $signedFilesDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force - - $itemsToCopyWithRecurse = @( - "$($fullSymbolsFolder)\*.ps1" - "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" - ) - - $itemsToCopy = @{ - "$($fullSymbolsFolder)\*.ps1" = "" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Host\Microsoft.PowerShell.Host.psd1" = "Modules\Microsoft.PowerShell.Host" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Management\Microsoft.PowerShell.Management.psd1" = "Modules\Microsoft.PowerShell.Management" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Microsoft.PowerShell.Security.psd1" = "Modules\Microsoft.PowerShell.Security" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Utility\Microsoft.PowerShell.Utility.psd1" = "Modules\Microsoft.PowerShell.Utility" - "$($fullSymbolsFolder)\pwsh.dll" = "" - "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" - } - - ## Windows only modules - - if('${{ parameters.addWindowsModules }}' -ne 'false') { - $itemsToCopy += @{ - "$($fullSymbolsFolder)\pwsh.exe" = "" - "$($fullSymbolsFolder)\Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" - "$($fullSymbolsFolder)\Microsoft.WSMan.*.dll" = "" - "$($fullSymbolsFolder)\Modules\CimCmdlets\CimCmdlets.psd1" = "Modules\CimCmdlets" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Security.types.ps1xml" = "Modules\Microsoft.PowerShell.Security" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Microsoft.PowerShell.Diagnostics.psd1" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" - "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" - "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" - } - } - else { - $itemsToCopy += @{ - "$($fullSymbolsFolder)\pwsh" = "" - } - } - - $itemsToExclude = @( - # This package is retrieved from https://www.github.com/powershell/MarkdownRender - "$($fullSymbolsFolder)\Microsoft.PowerShell.MarkdownRender.dll" - ) - - Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" - Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude - - foreach($pattern in $itemsToCopy.Keys) { - $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern - $null = New-Item -ItemType Directory -Path $destinationFolder -Force - Write-Verbose -verbose "copying $pattern to $destinationFolder" - Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose - } - displayName: '${{ parameters.buildPrefixName }} - Prepare files to be signed' - -- template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\toBeSigned - signOutputPath: $(System.ArtifactsDirectory)\signed - certificateId: "$(AUTHENTICODE_CERT)" - pattern: | - **\*.dll - **\*.psd1 - **\*.psm1 - **\*.ps1xml - **\*.ps1 - **\*.exe - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: ${{ parameters.buildPrefixName }} - Authenticode - -- pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' - $BuildPath = Join-Path $(System.ArtifactsDirectory) '${{ parameters.binLocation }}' - Write-Verbose -Verbose -Message "BuildPath: $BuildPath" - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - $dlls = Get-ChildItem $BuildPath\*.dll, $BuildPath\*.exe -Recurse - $signatures = $dlls | Get-AuthenticodeSignature - $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch '^CN=Microsoft.*'}| select-object -ExpandProperty Path - - Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" - - $filesToSignDirectory = "$(System.ArtifactsDirectory)\thirdPartyToBeSigned" - if (Test-Path $filesToSignDirectory) { - Remove-Item -Path $filesToSignDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force -Verbose - - $signedFilesDirectory = "$(System.ArtifactsDirectory)\thirdPartySigned" - if (Test-Path $signedFilesDirectory) { - Remove-Item -Path $signedFilesDirectory -Recurse -Force - } - - $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force -Verbose - - $missingSignatures | ForEach-Object { - $pathWithoutLeaf = Split-Path $_ - $relativePath = $pathWithoutLeaf.replace($BuildPath,'') - Write-Verbose -Verbose -Message "relativePath: $relativePath" - $targetDirectory = Join-Path -Path $filesToSignDirectory -ChildPath $relativePath - Write-Verbose -Verbose -Message "targetDirectory: $targetDirectory" - if(!(Test-Path $targetDirectory)) - { - $null = New-Item -ItemType Directory -Path $targetDirectory -Force -Verbose - } - Copy-Item -Path $_ -Destination $targetDirectory - } - - displayName: ${{ parameters.buildPrefixName }} - Create ThirdParty Signing Folder - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - -- template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\thirdPartyToBeSigned - signOutputPath: $(System.ArtifactsDirectory)\thirdPartySigned - certificateId: "CP-231522" - pattern: | - **\*.dll - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign ThirdParty binaries - -- pwsh: | - Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' - displayName: ${{ parameters.buildPrefixName }} - Capture ThirdParty Signed files - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - -- pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' - $BuildPath = Join-Path $(System.ArtifactsDirectory) '${{ parameters.binLocation }}' - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - if ($env:BuildConfiguration -eq 'minSize') { - ## Remove XML files when making a min-size package. - Remove-Item "$BuildPath/*.xml" -Force - } - displayName: ${{ parameters.buildPrefixName }} - Merge ThirdParty signed files with Build - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - -- pwsh: | - $uploadFolder = '$(System.ArtifactsDirectory)/${{ parameters.binLocation }}' - $containerName = 'authenticode-signed' - - Write-Verbose -Verbose "File permissions after signing" - Get-ChildItem $uploadFolder\pwsh | Select-Object -Property 'unixmode', 'size', 'name' - - $uploadTarFilePath = '$(System.ArtifactsDirectory)/${{ parameters.binLocation }}.tar.gz' - Write-Verbose -Verbose -Message "Creating tar.gz - $uploadTarFilePath" - tar -czvf $uploadTarFilePath -C $uploadFolder * - - Write-Host "##vso[artifact.upload containerfolder=$containerName;artifactname=$containerName]$uploadTarFilePath" - displayName: ${{ parameters.buildPrefixName }} - Upload signed files to artifacts - retryCountOnTaskFailure: 2 - diff --git a/tools/releaseBuild/azureDevOps/templates/step/finalize.yml b/tools/releaseBuild/azureDevOps/templates/step/finalize.yml deleted file mode 100644 index 72a677fec9a..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/step/finalize.yml +++ /dev/null @@ -1,5 +0,0 @@ -steps: - - pwsh: | - throw "Jobs with an Issue will not work for release. Please fix the issue and try again." - displayName: Check for SucceededWithIssues - condition: eq(variables['Agent.JobStatus'],'SucceededWithIssues') diff --git a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml deleted file mode 100644 index 43c09236da9..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml +++ /dev/null @@ -1,126 +0,0 @@ -jobs: -- job: build_testartifacts_win - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: DotNetPrivateBuildAccess - displayName: Build windows test artifacts - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - steps: - - checkout: self - clean: true - - - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(Build.SourcesDirectory) - - - pwsh: | - Import-Module ./build.psm1 - Start-PSBootstrap - displayName: Bootstrap - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - Import-Module ./build.psm1 - - function BuildTestPackage([string] $runtime) - { - Write-Verbose -Verbose "Starting to build package for $runtime" - - New-TestPackage -Destination $(System.ArtifactsDirectory) -Runtime $runtime - - if (-not (Test-Path $(System.ArtifactsDirectory)/TestPackage.zip)) - { - throw "Test Package was not found at: $(System.ArtifactsDirectory)" - } - - switch ($runtime) - { - win7-x64 { $packageName = "TestPackage-win-x64.zip" } - win7-x86 { $packageName = "TestPackage-win-x86.zip" } - win-arm64 { $packageName = "TestPackage-win-arm64.zip" } - } - - Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName - Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" - } - - BuildTestPackage -runtime win7-x64 - BuildTestPackage -runtime win7-x86 - BuildTestPackage -runtime win-arm64 - - displayName: Build test package and upload - retryCountOnTaskFailure: 1 - -- job: build_testartifacts_nonwin - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - group: DotNetPrivateBuildAccess - displayName: Build non-windows test artifacts - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMSUbuntu20.04-Secure - steps: - - checkout: self - clean: true - - - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(Build.SourcesDirectory) - - - pwsh: | - Import-Module ./build.psm1 - Start-PSBootstrap - displayName: Bootstrap - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - Import-Module ./build.psm1 - - function BuildTestPackage([string] $runtime) - { - Write-Verbose -Verbose "Starting to build package for $runtime" - - New-TestPackage -Destination $(System.ArtifactsDirectory) -Runtime $runtime - - if (-not (Test-Path $(System.ArtifactsDirectory)/TestPackage.zip)) - { - throw "Test Package was not found at: $(System.ArtifactsDirectory)" - } - - switch ($runtime) - { - linux-x64 { $packageName = "TestPackage-linux-x64.zip" } - linux-arm { $packageName = "TestPackage-linux-arm.zip" } - linux-arm64 { $packageName = "TestPackage-linux-arm64.zip" } - osx-x64 { $packageName = "TestPackage-macOS.zip" } - linux-musl-x64 { $packageName = "TestPackage-alpine-x64.zip"} - } - - Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName - Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" - } - - BuildTestPackage -runtime linux-x64 - BuildTestPackage -runtime linux-arm - BuildTestPackage -runtime linux-arm64 - BuildTestPackage -runtime osx-x64 - BuildTestPackage -runtime linux-musl-x64 - - displayName: Build test package and upload - retryCountOnTaskFailure: 1 - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml b/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml deleted file mode 100644 index 596b61fb6ed..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml +++ /dev/null @@ -1,17 +0,0 @@ -parameters: - artifactPath: - artifactFilter: '*' - condition: succeeded() - artifactName: finalResults - -steps: - - powershell: | - Get-ChildItem -Path '${{ parameters.artifactPath }}' -Recurse -File -filter '${{ parameters.artifactFilter }}' -ErrorAction SilentlyContinue | - Select-Object -ExpandProperty FullName | - ForEach-Object { - Write-Host "##vso[artifact.upload containerfolder=${{ parameters.artifactName }};artifactname=${{ parameters.artifactName }}]$_" - } - displayName: Upload ${{ parameters.artifactName }} Artifacts ${{ parameters.artifactFilter }} from ${{ parameters.artifactPath }} - condition: ${{ parameters.condition }} - retryCountOnTaskFailure: 2 - diff --git a/tools/releaseBuild/azureDevOps/templates/upload.yml b/tools/releaseBuild/azureDevOps/templates/upload.yml deleted file mode 100644 index c745a02c2a4..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/upload.yml +++ /dev/null @@ -1,83 +0,0 @@ -parameters: - architecture: x86 - version: 6.2.0 - msi: yes - msix: yes - pdb: no - -steps: -- template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\signed - artifactFilter: PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}*.zip - -- task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - resourceGroup: '$(StorageResourceGroup)' - condition: succeeded() - retryCountOnTaskFailure: 2 - -- task: AzureFileCopy@4 - displayName: 'upload signed min-size package (for Guest Config) to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}-gc.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)-gc' - resourceGroup: '$(StorageResourceGroup)' - condition: and(eq('${{ parameters.architecture }}', 'x64'), succeeded()) - retryCountOnTaskFailure: 2 - -- template: upload-final-results.yml - parameters: - artifactPath: $(System.ArtifactsDirectory)\signedPackages - artifactFilter: PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.exe - condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) - -- task: AzureFileCopy@4 - displayName: 'upload signed exe to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signedPackages\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.exe' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)-private' - resourceGroup: '$(StorageResourceGroup)' - condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) - retryCountOnTaskFailure: 2 - -# Disable upload task as the symbols package is not currently used and we want to avoid publishing this in releases -#- task: AzureFileCopy@4 -# displayName: 'upload pbd zip to Azure - ${{ parameters.architecture }}' -# inputs: -# SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-Symbols-${{ parameters.version }}-win-${{ parameters.architecture }}.zip' -# azureSubscription: '$(AzureFileCopySubscription)' -# Destination: AzureBlob -# storage: '$(StorageAccount)' -# ContainerName: '$(AzureVersion)' -# condition: and(succeeded(), eq('${{ parameters.pdb }}', 'yes')) - -- template: upload-final-results.yml - parameters: - artifactPath: $(Build.StagingDirectory)\signedPackages - artifactFilter: PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.msix - condition: and(succeeded(), eq('${{ parameters.msix }}', 'yes')) - -- task: AzureFileCopy@4 - displayName: 'upload signed msix to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.msix' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)-private' - resourceGroup: '$(StorageResourceGroup)' - condition: and(succeeded(), eq('${{ parameters.msix }}', 'yes'), eq(variables['SHOULD_SIGN'], 'true')) - retryCountOnTaskFailure: 2 diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml deleted file mode 100644 index 83779c75aa0..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ /dev/null @@ -1,113 +0,0 @@ -parameters: - architecture: x64 - -jobs: -- job: vpack_${{ parameters.architecture }} - variables: - - group: vPack - - group: ReleasePipelineSecrets - - displayName: Build and Publish VPack - ${{ parameters.architecture }} - condition: succeeded() - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - steps: - - checkout: self - clean: true - - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk' - inputs: - packageType: sdk - version: 3.1.x - installationPath: $(Agent.ToolsDirectory)/dotnet - - - template: ./SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - Install-AzCopy - displayName: Install AzCopy - retryCountOnTaskFailure: 2 - - - pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - $azcopy = Find-AzCopy - Write-Verbose -Verbose "Found AzCopy: $azcopy" - - Write-Host "running: $azcopy cp https://$(StorageAccount).blob.core.windows.net/$(AzureVersion)/PowerShell-$(Version)-win-${{ parameters.architecture }}.zip $(System.ArtifactsDirectory)" - - & $azcopy cp https://$(StorageAccount).blob.core.windows.net/$(AzureVersion)/PowerShell-$(Version)-win-${{ parameters.architecture }}.zip $(System.ArtifactsDirectory) - displayName: 'Download Azure Artifacts' - retryCountOnTaskFailure: 2 - env: - AZCOPY_AUTO_LOGIN_TYPE: MSI - - - pwsh: 'Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name' - displayName: 'Capture Artifact Listing' - - - pwsh: | - $message = @() - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -include *.zip, *.msi | ForEach-Object { - if($_.Name -notmatch 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(fxdependent|x64|arm64|x86|fxdependentWinDesktop)\.(msi|zip){1}') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } - } - - if($message.count -gt 0){throw ($message | out-string)} - displayName: 'Validate Zip and MSI Package Names' - - - pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -include *.zip, *.msi | ForEach-Object { - if($_.Name -match 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(${{ parameters.architecture }})\.(zip){1}') - { - $destDir = "$(System.ArtifactsDirectory)\vpack${{ parameters.architecture }}" - $null = new-item -ItemType Directory -Path $destDir - Expand-Archive -Path $_.FullName -DestinationPath $destDir - $vstsCommandString = "vso[task.setvariable variable=vpackDir]$destDir" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - } - } - displayName: 'Extract Zip' - - - pwsh: | - $vpackVersion = '$(version)' - - if('$(VPackPublishOverride)' -ne '' -and '$(VPackPublishOverride)' -ne 'None' ) - { - Write-Host "Using VPackPublishOverride varabile" - $vpackVersion = '$(VPackPublishOverride)' - } - - $vstsCommandString = "vso[task.setvariable variable=vpackVersion]$vpackVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set vpackVersion' - - - pwsh: | - Get-ChildItem -Path env: - displayName: Capture Environment - condition: succeededOrFailed() - - - task: PkgESVPack@12 - displayName: 'Package ES - VPack ' - inputs: - sourceDirectory: '$(vpackDir)' - description: PowerShell ${{ parameters.architecture }} $(version) - pushPkgName: 'PowerShell.${{ parameters.architecture }}' - configurations: Release - platforms: x64 - target: '$(System.ArtifactsDirectory)' - owner: tplunk - provData: true - version: '$(vpackVersion)' - vpackToken: $(vPackPat) - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml deleted file mode 100644 index 53947655d90..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ /dev/null @@ -1,71 +0,0 @@ - -jobs: -- job: ComponentRegistrationJob - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - displayName: Component Registration - condition: succeeded() - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: self - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - displayName: 'Capture Docker Info' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - template: insert-nuget-config-azfeed.yml - - - powershell: | - ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-x64-component-registration - displayName: 'Build Windows Universal - Component Registration' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(componentregistration)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml deleted file mode 100644 index 4b36f6f396e..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ /dev/null @@ -1,84 +0,0 @@ -parameters: - - name: BuildConfiguration - default: release - - name: BuildPlatform - default: any cpu - - name: Architecture - default: x64 - - name: parentJob - default: '' - -jobs: -- job: build_windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} - displayName: Build Windows - ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} - condition: succeeded() - dependsOn: ${{ parameters.parentJob }} - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: BuildConfiguration - value: ${{ parameters.BuildConfiguration }} - - name: BuildPlatform - value: ${{ parameters.BuildPlatform }} - - name: Architecture - value: ${{ parameters.Architecture }} - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: DotNetPrivateBuildAccess - - steps: - - - checkout: self - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: cloneToOfficialPath.yml - - - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - parameters: - repoRoot: $(PowerShellRoot) - - - pwsh: | - - $runtime = switch ($env:Architecture) - { - "x64" { "win7-x64" } - "x86" { "win7-x86" } - "arm64" { "win-arm64" } - "fxdependent" { "fxdependent" } - "fxdependentWinDesktop" { "fxdependent-win-desktop" } - } - - $params = @{} - if ($env:BuildConfiguration -eq 'minSize') { - $params['ForMinimalSize'] = $true - } - - tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -location '$(PowerShellRoot)' -destination '$(Build.ArtifactStagingDirectory)/Symbols_$(Architecture)' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' -Symbols @params - displayName: 'Build Windows Universal - $(Architecture)-$(BuildConfiguration) Symbols zip' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - $packageName = (Get-ChildItem '$(Build.ArtifactStagingDirectory)\Symbols_$(Architecture)').FullName - $vstsCommandString = "vso[artifact.upload containerfolder=results;artifactname=results]$packageName" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Upload symbols package - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(PowerShellRoot)\tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml deleted file mode 100644 index 75153ce0592..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ /dev/null @@ -1,132 +0,0 @@ -parameters: - parentJobs: [] - -jobs: -- job: WinPackageSigningJob - displayName: Windows Package signing and upload - dependsOn: - ${{ parameters.parentJobs }} - condition: succeeded() - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - variables: - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: ESRP - - name: repoFolder - value: PowerShell - - name: repoRoot - value: $(Agent.BuildDirectory)\$(repoFolder) - - name: complianceRepoFolder - value: compliance - - steps: - - checkout: self - clean: true - path: $(repoFolder) - - - checkout: ComplianceRepo - clean: true - path: $(complianceRepoFolder) - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: shouldSign.yml - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifacts' - inputs: - buildType: current - downloadType: single - artifactName: signed - downloadPath: '$(System.ArtifactsDirectory)' - - - powershell: | - dir "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture Downloaded Artifacts' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\signed - signOutputPath: $(Build.StagingDirectory)\signedPackages - certificateId: $(MSIX_CERT) - pattern: | - **\*.msix - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign msix - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\signed - signOutputPath: $(Build.StagingDirectory)\signedPackages - certificateId: $(AUTHENTICODE_CERT) - pattern: | - **\*.exe - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign exe - - - powershell: | - new-item -itemtype Directory -path '$(Build.StagingDirectory)\signedPackages' - Get-ChildItem "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-*.msi*" | copy-item -Destination '$(Build.StagingDirectory)\signedPackages' - displayName: 'Fake msi* Signing' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-*.exe" | copy-item -Destination '$(Build.StagingDirectory)\signedPackages' - displayName: 'Fake exe Signing' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - - - template: upload.yml - parameters: - architecture: x86 - version: $(version) - - - template: upload.yml - parameters: - architecture: x64 - version: $(version) - pdb: yes - - - template: upload.yml - parameters: - architecture: arm64 - version: $(version) - msi: yes - - - template: upload.yml - parameters: - architecture: fxdependent - version: $(version) - msi: no - msix: no - - - template: upload.yml - parameters: - architecture: fxdependentWinDesktop - version: $(version) - msi: no - msix: no - - - template: EsrpScan.yml@ComplianceRepo - parameters: - scanPath: $(Build.StagingDirectory) - pattern: | - **\*.msix - **\*.msi - **\*.zip - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(repoRoot)\tools' - snapshotForceEnabled: true - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml deleted file mode 100644 index 915db9301ac..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ /dev/null @@ -1,369 +0,0 @@ -parameters: - - name: BuildConfiguration - default: release - - name: BuildPlatform - default: any cpu - - name: Architecture - default: x64 - - name: parentJob - default: '' - -jobs: -- job: sign_windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} - displayName: Package Windows - ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} - condition: succeeded() - pool: - name: $(windowsPool) - demands: - - ImageOverride -equals PSMMS2019-Secure - variables: - - name: BuildConfiguration - value: ${{ parameters.BuildConfiguration }} - - name: BuildPlatform - value: ${{ parameters.BuildPlatform }} - - name: Architecture - value: ${{ parameters.Architecture }} - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: ESRP - - group: DotNetPrivateBuildAccess - - steps: - - - checkout: self - clean: true - - - checkout: ComplianceRepo - clean: true - - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - template: shouldSign.yml - - - pwsh: | - $pkgFilter = '$(Architecture)' - if ($env:BuildConfiguration -eq 'minSize') { $pkgFilter += '-gc' } - - $vstsCommandString = "vso[task.setvariable variable=PkgFilter]$pkgFilter" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Set packageName variable - - - task: DownloadBuildArtifacts@0 - inputs: - artifactName: 'results' - itemPattern: '**/*$(PkgFilter).zip' - downloadPath: '$(System.ArtifactsDirectory)\Symbols' - - - template: cloneToOfficialPath.yml - - - pwsh: | - $zipPathString = '$(System.ArtifactsDirectory)\Symbols\results\*$(PkgFilter).zip' - Write-Verbose -Verbose "Zip Path: $zipPathString" - $zipPath = Get-Item $zipPathString - if(@($zipPath).Count -eq 0) { - throw "No files found at '$zipPathString'" - } - elseif(@($zipPath).Count -ne 1) { - $names = $zipPath.Name -join "', '" - throw "multiple files '${names}' found with '${zipPathString}'" - } - - $expandedFolder = $zipPath.BaseName - Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" - Write-Host "##vso[task.setvariable variable=SymbolsFolder]$expandedFolder" - - Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force - displayName: Expand symbols zip - - - pwsh: | - $fullSymbolsFolder = "$(System.ArtifactsDirectory)\$($env:SYMBOLSFOLDER)" - - $filesToSignDirectory = "$(System.ArtifactsDirectory)\toBeSigned" - $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force - - $signedFilesDirectory = "$(System.ArtifactsDirectory)\signed" - $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force - - $itemsToCopyWithRecurse = @( - "$($fullSymbolsFolder)\*.ps1" - "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" - ) - - $itemsToCopy = @{ - "$($fullSymbolsFolder)\*.ps1" = "" - "$($fullSymbolsFolder)\Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" - "$($fullSymbolsFolder)\Microsoft.WSMan.*.dll" = "" - "$($fullSymbolsFolder)\Modules\CimCmdlets\CimCmdlets.psd1" = "Modules\CimCmdlets" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Microsoft.PowerShell.Diagnostics.psd1" = "Modules\Microsoft.PowerShell.Diagnostics" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Host\Microsoft.PowerShell.Host.psd1" = "Modules\Microsoft.PowerShell.Host" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Management\Microsoft.PowerShell.Management.psd1" = "Modules\Microsoft.PowerShell.Management" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Microsoft.PowerShell.Security.psd1" = "Modules\Microsoft.PowerShell.Security" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Security.types.ps1xml" = "Modules\Microsoft.PowerShell.Security" - "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Utility\Microsoft.PowerShell.Utility.psd1" = "Modules\Microsoft.PowerShell.Utility" - "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" - "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" - "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" - "$($fullSymbolsFolder)\pwsh.dll" = "" - "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" - "$($fullSymbolsFolder)\pwsh.exe" = "" - } - - $itemsToExclude = @( - # This package is retrieved from https://www.github.com/powershell/MarkdownRender - "$($fullSymbolsFolder)\Microsoft.PowerShell.MarkdownRender.dll" - ) - - Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" - Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude - - foreach($pattern in $itemsToCopy.Keys) { - $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern - $null = New-Item -ItemType Directory -Path $destinationFolder -Force - Write-Verbose -verbose "copying $pattern to $destinationFolder" - Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose - } - displayName: 'Prepare files to be signed' - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\toBeSigned - signOutputPath: $(System.ArtifactsDirectory)\signed - certificateId: "$(AUTHENTICODE_CERT)" - pattern: | - **\*.dll - **\*.psd1 - **\*.psm1 - **\*.ps1xml - **\*.ps1 - **\*.exe - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign our binaries - - - pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' - $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - $dlls = Get-ChildItem $BuildPath\*.dll, $BuildPath\*.exe -Recurse - $signatures = $dlls | Get-AuthenticodeSignature - $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch '^CN=Microsoft.*'}| select-object -ExpandProperty Path - - Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" - - $filesToSignDirectory = "$(System.ArtifactsDirectory)\thirdPartyToBeSigned" - $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force - - $signedFilesDirectory = "$(System.ArtifactsDirectory)\thirdPartySigned" - $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force - - $missingSignatures | ForEach-Object { - $pathWithoutLeaf = Split-Path $_ - $relativePath = $pathWithoutLeaf.replace($BuildPath,'') - $targetDirectory = Join-Path -Path $filesToSignDirectory -ChildPath $relativePath - if(!(Test-Path $targetDirectory)) - { - $null = New-Item -ItemType Directory -Path $targetDirectory -Force - } - Copy-Item -Path $_ -Destination $targetDirectory - } - - displayName: Create ThirdParty Signing Folder - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\thirdPartyToBeSigned - signOutputPath: $(System.ArtifactsDirectory)\thirdPartySigned - certificateId: "CP-231522" - pattern: | - **\*.dll - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign ThirdParty binaries - - - pwsh: | - Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' - displayName: Capture ThirdParty Signed files - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' - $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath - if ($env:BuildConfiguration -eq 'minSize') { - ## Remove XML files when making a min-size package. - Remove-Item "$BuildPath/*.xml" -Force - } - displayName: Merge ThirdParty signed files with Build - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - PackageName: PowerShell Windows ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)\tools' - - - pwsh: | - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - - $destFolder = '$(System.ArtifactsDirectory)\signedZip' - $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - - New-Item -ItemType Directory -Path $destFolder -Force - - $BuildPackagePath = New-PSBuildZip -BuildPath $BuildPath -DestinationFolder $destFolder - - Write-Verbose -Verbose "New-PSSignedBuildZip returned `$BuildPackagePath as: $BuildPackagePath" - Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$BuildPackagePath" - - $vstsCommandString = "vso[task.setvariable variable=BuildPackagePath]$BuildPackagePath" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Compress signed files - retryCountOnTaskFailure: 2 - - - - pwsh: | - $runtime = switch ($env:Architecture) - { - "x64" { "win7-x64" } - "x86" { "win7-x86" } - "arm64" { "win-arm64" } - "fxdependent" { "fxdependent" } - "fxdependentWinDesktop" { "fxdependent-win-desktop" } - } - - $signedPkg = "$(BuildPackagePath)" - Write-Verbose -Verbose -Message "signedPkg = $signedPkg" - - $params = @{} - if ($env:BuildConfiguration -eq 'minSize') { - $params['ForMinimalSize'] = $true - } - - $(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -BuildZip $signedPkg -location '$(PowerShellRoot)' -destination '$(System.ArtifactsDirectory)\pkgSigned' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' @params - displayName: 'Build Windows Universal - $(Architecture) Package' - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - - pwsh: | - Get-ChildItem '$(System.ArtifactsDirectory)\pkgSigned' | ForEach-Object { - $packagePath = $_.FullName - Write-Host "Uploading $packagePath" - Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$packagePath" - } - displayName: Upload unsigned packages - retryCountOnTaskFailure: 2 - - - ${{ if and(ne(variables['BuildConfiguration'],'minSize'), in(variables['Architecture'], 'x64', 'x86', 'arm64')) }}: - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\pkgSigned - signOutputPath: $(Build.StagingDirectory)\signedPackages - certificateId: "$(AUTHENTICODE_CERT)" - pattern: | - **\*.msi - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign MSI - alwaysCopy: true - - - pwsh: | - Get-ChildItem '$(System.ArtifactsDirectory)\signedPackages' | ForEach-Object { - $packagePath = $_.FullName - Write-Host "Uploading $packagePath" - Write-Host "##vso[artifact.upload containerfolder=finalResults;artifactname=finalResults]$packagePath" - } - displayName: Upload signed MSI to finalResults - retryCountOnTaskFailure: 2 - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(version)-win-${{ parameters.architecture }}.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - resourceGroup: '$(StorageResourceGroup)' - retryCountOnTaskFailure: 2 - - - pwsh: | - cd $(PowerShellRoot) - Import-Module $(PowerShellRoot)/build.psm1 -Force - Import-Module $(PowerShellRoot)/tools/packaging -Force - - $msiPath = '$(Build.StagingDirectory)\signedPackages\PowerShell-$(version)-win-${{ parameters.architecture }}.msi' - - New-ExePackage -ProductVersion '$(version)' -MsiLocationPath $msiPath -ProductTargetArchitecture ${{ parameters.architecture }} - $exePath = Get-ChildItem '.\PowerShell-*.exe' | Select-Object -First 1 -ExpandProperty fullname - $enginePath = Join-Path -Path '$(System.ArtifactsDirectory)\unsignedEngine' -ChildPath engine.exe - # Expand Burn Engine so we can sign it. - Expand-ExePackageEngine -ExePath $exePath -EnginePath $enginePath - displayName: Create exe wrapper - - - template: EsrpSign.yml@ComplianceRepo - parameters: - buildOutputPath: $(System.ArtifactsDirectory)\unsignedEngine - signOutputPath: $(System.ArtifactsDirectory)\signedEngine - certificateId: "$(AUTHENTICODE_CERT)" - pattern: | - **\*.exe - useMinimatch: true - shouldSign: $(SHOULD_SIGN) - displayName: Sign Burn Engine - alwaysCopy: true - - - pwsh: | - cd '$(PowerShellRoot)' - Import-Module '$(PowerShellRoot)/build.psm1' -Force - Import-Module '$(PowerShellRoot)/tools/packaging' -Force - - $exePath = Get-ChildItem '.\PowerShell-*.exe' | Select-Object -First 1 -ExpandProperty fullname - $enginePath = Join-Path -Path '$(System.ArtifactsDirectory)\signedEngine' -ChildPath engine.exe - $enginePath | Get-AuthenticodeSignature | out-string | Write-Verbose -verbose - Compress-ExePackageEngine -ExePath $exePath -EnginePath $enginePath - displayName: Re-attach the signed Burn engine in exe wrapper - - - pwsh: | - cd '$(PowerShellRoot)' - Get-ChildItem '.\PowerShell-*.exe' | ForEach-Object { - $packagePath = $_.FullName - Write-Host "Uploading $packagePath" - Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$packagePath" - } - displayName: Upload unsigned exe - retryCountOnTaskFailure: 2 - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(PowerShellRoot)\tools' - snapshotForceEnabled: true - - - pwsh: | - if ((Test-Path "\PowerShell")) { - Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose - } - else { - Write-Verbose -Verbose -Message "No cleanup required." - } - displayName: Clean up local Clone - condition: always() - - - template: /tools/releaseBuild/azureDevOps/templates/step/finalize.yml diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml deleted file mode 100644 index 14368ffb8f8..00000000000 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: vpack-$(Build.BuildId) -trigger: - branches: - include: - - master - - release* -pr: - branches: - include: - - master - - release* - -variables: - - name: DOTNET_CLI_TELEMETRY_OPTOUT - value: 1 - - name: POWERSHELL_TELEMETRY_OPTOUT - value: 1 - - name: nugetMultiFeedWarnLevel - value: none - - - group: Azure Blob variable group - # adds the pat to publish the vPack - # instructions to create are in the description of the library - - group: vPack - -stages: -- stage: prep - displayName: Create buildInfo and name the Pipeline - jobs: - - job: rename - displayName: Name the build - condition: succeeded() - - pool: - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMS2019-Secure - - steps: - - checkout: self - clean: true - - - template: ./templates/SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - UseJson: no - - - powershell: | - if($env:RELEASETAGVAR -match '-') { - throw "Don't release a preview build without coordinating with Windows Engineering Build Tools Team" - } - displayName: Stop any preview release - - - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))" - displayName: Set Build Name for Non-PR - condition: ne(variables['Build.Reason'], 'PullRequest') - -- stage: release - displayName: Release - jobs: - - template: ./templates/vpackReleaseJob.yml - parameters: - architecture: x64 - - - template: ./templates/vpackReleaseJob.yml - parameters: - architecture: x86 - - - template: ./templates/vpackReleaseJob.yml - parameters: - architecture: arm64 diff --git a/tools/releaseBuild/build.json b/tools/releaseBuild/build.json deleted file mode 100644 index fe2f9d96f17..00000000000 --- a/tools/releaseBuild/build.json +++ /dev/null @@ -1,336 +0,0 @@ -{ - "Windows": [ - { - "Name": "win7-x64", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "release", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win7-x86", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "release", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-x64-component-registration", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_ -ComponentRegistration", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "componentregistration", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-x64-symbols", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_ -Symbols", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "Symbols_x64", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-x86-symbols", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_ -Symbols", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "Symbols_x86", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-arm-symbols", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm -ReleaseTag _ReleaseTag_ -Symbols", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "Symbols_arm", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-arm64-symbols", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm64 -ReleaseTag _ReleaseTag_ -Symbols", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "Symbols_arm64", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-x64-package", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "signed", - "ArtifactsExpected": 4, - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-x86-package", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "signed", - "ArtifactsExpected": 4, - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-arm-package", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "signed", - "ArtifactsExpected": 2, - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-arm64-package", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm64 -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "signed", - "ArtifactsExpected": 2, - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-fxdependent-symbols", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime fxdependent -ReleaseTag _ReleaseTag_ -Symbols", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "Symbols_fxdependent", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-fxdependent-package", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime fxdependent -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "signed", - "ArtifactsExpected": 1, - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-fxdependentWinDesktop-symbols", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime fxdependent-win-desktop -ReleaseTag _ReleaseTag_ -Symbols", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "results", - "ArtifactsExpected": 1, - "VariableForExtractedBinariesPath": "Symbols_fxdependentWinDesktop", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "win-fxdependentWinDesktop-package", - "RepoDestinationPath": "C:\\PowerShell", - "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime fxdependent-win-desktop -ReleaseTag _ReleaseTag_", - "BuildDockerOptions": [ - "-m", - "3968m" - ], - "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\Dockerfile", - "AdditionalContextFiles" :[ - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", - ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" - ], - "DockerImageName": "ps-winsrvcore", - "BinaryBucket": "signed", - "ArtifactsExpected": 1, - "EnableFeature": [ "ArtifactAsFolder" ] - } - ], - "Linux": [ - { - "Name": "deb", - "RepoDestinationPath": "/PowerShell", - "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_ -TarX64 -TarArm -TarArm64 -TarMinSize", - "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_ubuntu18.04/Dockerfile", - "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], - "DockerImageName": "ps-ubunutu-18-04", - "BinaryBucket": "release", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "rpm", - "RepoDestinationPath": "/PowerShell", - "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_", - "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], - "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile", - "DockerImageName": "ps-centos-7", - "BinaryBucket": "release", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "alpine", - "RepoDestinationPath": "/PowerShell", - "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_ -Alpine", - "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], - "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile", - "DockerImageName": "ps-alpine-3", - "BinaryBucket": "release", - "EnableFeature": [ "ArtifactAsFolder" ] - }, - { - "Name": "fxdependent", - "RepoDestinationPath": "/PowerShell", - "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_ -FxDependent", - "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], - "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile", - "DockerImageName": "ps-centos-7", - "BinaryBucket": "release", - "EnableFeature": [ "ArtifactAsFolder" ] - } - ] -} diff --git a/tools/releaseBuild/createComplianceFolder.ps1 b/tools/releaseBuild/createComplianceFolder.ps1 deleted file mode 100644 index c462a09ebdb..00000000000 --- a/tools/releaseBuild/createComplianceFolder.ps1 +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -param( - [Parameter(HelpMessage="Artifact folder to find compliance files in.")] - [string[]] - $ArtifactFolder, - [Parameter(HelpMessage="VSTS Variable to set path to complinance Files.")] - [string] - $VSTSVariableName -) - -$compliancePath = $null -foreach($folder in $ArtifactFolder) -{ - # Find Symbols zip which contains compliance files - Write-Host "ArtifactFolder: $folder" - $filename = Join-Path -Path $folder -ChildPath 'symbols.zip' - - $parentName = Split-Path -Path $folder -Leaf - - # Use simplified names because some of the compliance tools didn't like the full names - # decided not to use hashes because the names need to be consistent otherwise the tool also has issues - # which is another problem with the full name, it includes version. - if ($parentName -match 'x64' -or $parentName -match 'amd64') - { - $name = 'x64' - } - elseif ($parentName -match 'x86') { - $name = 'x86' - } - elseif ($parentName -match 'fxdependent') { - $name = 'fxd' - } - else - { - throw "$parentName could not be classified as x86 or x64" - } - - # Throw is compliance zip does not exist - if (!(Test-Path $filename)) - { - throw "symbols.zip for $VSTSVariableName does not exist" - } - - # make sure we have a single parent for everything - if (!$compliancePath) - { - $parent = Split-Path -Path $folder - $compliancePath = Join-Path -Path $parent -ChildPath 'compliance' - } - - # Extract complance files to individual folder to avoid overwriting files. - $unzipPath = Join-Path -Path $compliancePath -ChildPath $name - Write-Host "Symbols-zip: $filename ; unzipPath: $unzipPath" - Expand-Archive -Path $fileName -DestinationPath $unzipPath -} - -# set VSTS variable with path to compliance files -Write-Host "##vso[task.setvariable variable=$VSTSVariableName]$unzipPath" diff --git a/tools/releaseBuild/generatePackgeSigning.ps1 b/tools/releaseBuild/generatePackgeSigning.ps1 deleted file mode 100644 index ff848892097..00000000000 --- a/tools/releaseBuild/generatePackgeSigning.ps1 +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -param( - [Parameter(Mandatory)] - [string] $Path, - [string[]] $AuthenticodeDualFiles, - [string[]] $AuthenticodeFiles, - [string[]] $NuPkgFiles, - [string[]] $MacDeveloperFiles, - [string[]] $LinuxFiles, - [string[]] $ThirdPartyFiles, - [string[]] $MsixFiles, - [ValidateSet('release','preview')] - [string] $MsixCertType = 'preview' -) - -if ((!$AuthenticodeDualFiles -or $AuthenticodeDualFiles.Count -eq 0) -and - (!$AuthenticodeFiles -or $AuthenticodeFiles.Count -eq 0) -and - (!$NuPkgFiles -or $NuPkgFiles.Count -eq 0) -and - (!$MacDeveloperFiles -or $MacDeveloperFiles.Count -eq 0) -and - (!$LinuxFiles -or $LinuxFiles.Count -eq 0) -and - (!$MsixFiles -or $MsixFiles.Count -eq 0) -and - (!$ThirdPartyFiles -or $ThirdPartyFiles.Count -eq 0)) -{ - throw "At least one file must be specified" -} - -function New-Attribute -{ - param( - [Parameter(Mandatory)] - [string]$Name, - [Parameter(Mandatory)] - [object]$Value, - [Parameter(Mandatory)] - [System.Xml.XmlElement]$Element - ) - - $attribute = $signingXml.CreateAttribute($Name) - $attribute.Value = $value - $null = $fileElement.Attributes.Append($attribute) -} - -function New-FileElement -{ - param( - [Parameter(Mandatory)] - [string]$File, - [Parameter(Mandatory)] - [string]$SignType, - [Parameter(Mandatory)] - [System.Xml.XmlDocument]$XmlDoc, - [Parameter(Mandatory)] - [System.Xml.XmlElement]$Job - ) - - if(Test-Path -Path $file) - { - $name = Split-Path -Leaf -Path $File - $fileElement = $XmlDoc.CreateElement("file") - New-Attribute -Name 'src' -value $file -Element $fileElement - New-Attribute -Name 'signType' -value $SignType -Element $fileElement - New-Attribute -Name 'dest' -value "__OUTPATHROOT__\$name" -Element $fileElement - $null = $job.AppendChild($fileElement) - } - else - { - Write-Warning -Message "Skipping $SignType; $File because it does not exist" - } -} - -[xml]$signingXml = Get-Content (Join-Path -Path $PSScriptRoot -ChildPath 'packagesigning.xml') -$job = $signingXml.SignConfigXML.job - -foreach($file in $AuthenticodeDualFiles) -{ - New-FileElement -File $file -SignType 'AuthenticodeDual' -XmlDoc $signingXml -Job $job -} - -foreach($file in $AuthenticodeFiles) -{ - New-FileElement -File $file -SignType 'AuthenticodeFormer' -XmlDoc $signingXml -Job $job -} - -foreach($file in $NuPkgFiles) -{ - New-FileElement -File $file -SignType 'NuGet' -XmlDoc $signingXml -Job $job -} - -foreach ($file in $MacDeveloperFiles) { - New-FileElement -File $file -SignType 'MacDeveloper' -XmlDoc $signingXml -Job $job -} - -foreach ($file in $LinuxFiles) { - New-FileElement -File $file -SignType 'LinuxPack' -XmlDoc $signingXml -Job $job -} - -foreach ($file in $ThirdPartyFiles) { - New-FileElement -File $file -SignType 'ThirdParty' -XmlDoc $signingXml -Job $job -} - -foreach ($file in $MsixFiles) { - # 'CP-459155' is supposed to work for the store - # AuthenticodeFormer works for sideloading and via a workaround, through the store - # ---------------------------------------------- - # update releasePublisher in packaging.psm1 when this is changed - New-FileElement -File $file -SignType 'AuthenticodeFormer' -XmlDoc $signingXml -Job $job -} - -$signingXml.Save($path) -$updateScriptPath = Join-Path -Path $PSScriptRoot -ChildPath 'updateSigning.ps1' -& $updateScriptPath -SigningXmlPath $path diff --git a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 deleted file mode 100644 index acedbdd3388..00000000000 --- a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# PowerShell Script to build and package PowerShell from specified form and branch -# Script is intented to use in Docker containers -# Ensure PowerShell is available in the provided image - -param ( - # Set default location to where VSTS cloned the repository locally. - [string] $location = $env:BUILD_REPOSITORY_LOCALPATH, - - # Destination location of the package on docker host - [Parameter(Mandatory, ParameterSetName = 'packageSigned')] - [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] - [Parameter(Mandatory, ParameterSetName = 'Build')] - [string] $destination = '/mnt', - - [Parameter(Mandatory, ParameterSetName = 'packageSigned')] - [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] - [Parameter(Mandatory, ParameterSetName = 'Build')] - [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] - [ValidateNotNullOrEmpty()] - [string]$ReleaseTag, - - [Parameter(ParameterSetName = 'packageSigned')] - [Parameter(ParameterSetName = 'IncludeSymbols')] - [Parameter(ParameterSetName = 'Build')] - [ValidateSet("zip", "tar")] - [string[]]$ExtraPackage, - - [Parameter(Mandatory, ParameterSetName = 'Bootstrap')] - [switch] $BootStrap, - - [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] - [Parameter(Mandatory, ParameterSetName = 'Build')] - [switch] $Build, - - [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] - [switch] $Symbols, - - [Parameter(Mandatory, ParameterSetName = 'packageSigned')] - [ValidatePattern("-signed.zip$")] - [string]$BuildZip, - - [Parameter(Mandatory, ParameterSetName = 'packageSigned')] - [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] - [Parameter(Mandatory, ParameterSetName = 'Build')] - [ValidateSet('osx-x64', 'osx-arm64')] - [string]$Runtime, - - [string]$ArtifactName = 'result', - - [switch]$SkipReleaseChecks -) - -$repoRoot = $location - -if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { - $releaseTagParam = @{} - if ($ReleaseTag) { - $releaseTagParam['ReleaseTag'] = $ReleaseTag - - #Remove the initial 'v' from the ReleaseTag - $version = $ReleaseTag -replace '^v' - $semVersion = [System.Management.Automation.SemanticVersion] $version - - $metadata = Get-Content "$location/tools/metadata.json" -Raw | ConvertFrom-Json - - $LTS = $metadata.LTSRelease.Package - - Write-Verbose -Verbose -Message "LTS is set to: $LTS" - } -} - -Push-Location -try { - $pspackageParams = @{ SkipReleaseChecks = $SkipReleaseChecks; MacOSRuntime = $Runtime } - Write-Verbose -Message "Init..." -Verbose - Set-Location $repoRoot - Import-Module "$repoRoot/build.psm1" - Import-Module "$repoRoot/tools/packaging" - Sync-PSTags -AddRemoteIfMissing - - if ($BootStrap) { - Start-PSBootstrap -Package - } - - if ($PSCmdlet.ParameterSetName -eq 'packageSigned') { - Write-Verbose "Expanding signed build $BuildZip ..." -Verbose - Expand-PSSignedBuild -BuildZip $BuildZip - - Remove-Item -Path $BuildZip - - Start-PSPackage @pspackageParams @releaseTagParam - switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam } - } - - if ($LTS) { - Start-PSPackage @pspackageParams @releaseTagParam -LTS - switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam -LTS } - } - } - } - - if ($Build) { - if ($Symbols) { - Start-PSBuild -Clean -Configuration 'Release' -NoPSModuleRestore @releaseTagParam -Runtime $Runtime - $pspackageParams['Type']='zip' - $pspackageParams['IncludeSymbols']=$Symbols.IsPresent - Write-Verbose "Starting powershell packaging(zip)..." -Verbose - Start-PSPackage @pspackageParams @releaseTagParam - } else { - Start-PSBuild -Configuration 'Release' -PSModuleRestore @releaseTagParam -Runtime $Runtime - Start-PSPackage @pspackageParams @releaseTagParam - switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam } - } - - if ($LTS) { - Start-PSPackage @releaseTagParam -LTS - switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam -LTS } - } - } - } - } -} finally { - Pop-Location -} - -if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { - $macPackages = Get-ChildItem "$repoRoot/powershell*" -Include *.pkg, *.tar.gz, *.zip - foreach ($macPackage in $macPackages) { - $filePath = $macPackage.FullName - $extension = (Split-Path -Extension -Path $filePath).Replace('.', '') - Write-Verbose "Copying $filePath to $destination" -Verbose - Write-Host "##vso[artifact.upload containerfolder=$ArtifactName;artifactname=$ArtifactName]$filePath" - Write-Host "##vso[task.setvariable variable=Package-$extension]$filePath" - Copy-Item -Path $filePath -Destination $destination -Force - } -} diff --git a/tools/releaseBuild/macOS/PowerShellPackageVsts.sh b/tools/releaseBuild/macOS/PowerShellPackageVsts.sh deleted file mode 100644 index b7bfa7315d8..00000000000 --- a/tools/releaseBuild/macOS/PowerShellPackageVsts.sh +++ /dev/null @@ -1 +0,0 @@ -pwsh -command ".\PowerShellPackageVsts.ps1 $*" diff --git a/tools/releaseBuild/macOS/createPowerShell.sh b/tools/releaseBuild/macOS/createPowerShell.sh deleted file mode 100644 index 5b0b681716c..00000000000 --- a/tools/releaseBuild/macOS/createPowerShell.sh +++ /dev/null @@ -1,8 +0,0 @@ -# print version for diags -sw_vers -productVersion - -# create folder -sudo mkdir /PowerShell - -# make the current user the owner -sudo chown $USER /PowerShell diff --git a/tools/releaseBuild/packagesigning.xml b/tools/releaseBuild/packagesigning.xml deleted file mode 100644 index a243e5fbd98..00000000000 --- a/tools/releaseBuild/packagesigning.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/tools/releaseBuild/signing.xml b/tools/releaseBuild/signing.xml deleted file mode 100644 index a6b19f6a07a..00000000000 --- a/tools/releaseBuild/signing.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tools/releaseBuild/updateSigning.ps1 b/tools/releaseBuild/updateSigning.ps1 deleted file mode 100644 index bace3aec2b7..00000000000 --- a/tools/releaseBuild/updateSigning.ps1 +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -param( - [string] $SigningXmlPath = (Join-Path -Path $PSScriptRoot -ChildPath 'signing.xml'), - [switch] $SkipPwshExe -) -# Script for use in VSTS to update signing.xml - -if ($SkipPwshExe) { - ## This is required for fxdependent package as no .exe is generated. - $xmlContent = Get-Content $SigningXmlPath | Where-Object { $_ -notmatch '__INPATHROOT__\\pwsh.exe' } -} else { - ## We skip the global tool shim assembly for regular builds. - $xmlContent = Get-Content $signingXmlPath | Where-Object { $_ -notmatch '__INPATHROOT__\\Microsoft.PowerShell.GlobalTool.Shim.dll' } -} - -# Parse the signing xml -$signingXml = [xml] $xmlContent - -# Get any variables to updating 'signType' in the XML -# Define a varabile named `SignType' in VSTS to updating that signing type -# Example: $env:AuthenticodeSignType='newvalue' -# will cause all files with the 'Authenticode' signtype to be updated with the 'newvalue' signtype -$signTypes = @{} -Get-ChildItem -Path env:/*SignType | ForEach-Object -Process { - $signType = $_.Name.ToUpperInvariant().Replace('SIGNTYPE','') - Write-Host "Found SigningType $signType with value $($_.value)" - $signTypes[$signType] = $_.Value -} - -# examine each job in the xml -$signingXml.SignConfigXML.job | ForEach-Object -Process { - # examine each file in the job - $_.file | ForEach-Object -Process { - # if the sign type is one of the variables we found, update it to the new value - $signType = $_.SignType.ToUpperInvariant() - if($signTypes.ContainsKey($signType)) - { - $newSignType = $signTypes[$signType] - Write-Host "Updating $($_.src) to $newSignType" - $_.signType = $newSignType - } - } -} - -$signingXml.Save($signingXmlPath) diff --git a/tools/releaseBuild/vstsbuild.ps1 b/tools/releaseBuild/vstsbuild.ps1 deleted file mode 100644 index 1c2d740c418..00000000000 --- a/tools/releaseBuild/vstsbuild.ps1 +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -[cmdletbinding(DefaultParameterSetName='Build')] -param( - [Parameter(ParameterSetName='packageSigned')] - [Parameter(ParameterSetName='Build')] - [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] - [string]$ReleaseTag, - - # full paths to files to add to container to run the build - [Parameter(Mandatory,ParameterSetName='packageSigned')] - [string] - $BuildPath, - - [Parameter(Mandatory,ParameterSetName='packageSigned')] - [string] - $SignedFilesPath -) - -DynamicParam { - # Add a dynamic parameter '-Name' which specifies the name of the build to run - - # Get the names of the builds. - $buildJsonPath = (Join-Path -Path $PSScriptRoot -ChildPath 'build.json') - $build = Get-Content -Path $buildJsonPath | ConvertFrom-Json - $names = @($build.Windows.Name) - foreach($name in $build.Linux.Name) - { - $names += $name - } - - # Create the parameter attributs - $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute" - $ValidateSetAttr = New-Object "System.Management.Automation.ValidateSetAttribute" -ArgumentList $names - $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]" - $Attributes.Add($ParameterAttr) > $null - $Attributes.Add($ValidateSetAttr) > $null - - # Create the parameter - $Parameter = New-Object "System.Management.Automation.RuntimeDefinedParameter" -ArgumentList ("Name", [string], $Attributes) - $Dict = New-Object "System.Management.Automation.RuntimeDefinedParameterDictionary" - $Dict.Add("Name", $Parameter) > $null - return $Dict -} - -Begin { - $Name = $PSBoundParameters['Name'] -} - -End { - $ErrorActionPreference = 'Stop' - - $additionalFiles = @() - $buildPackageName = $null - # If specified, Add package file to container - if ($BuildPath) - { - Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '..\..\build.psm1') - Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '..\packaging') - - # Use temp as destination if not running in VSTS - $destFolder = $env:temp - if($env:BUILD_STAGINGDIRECTORY) - { - # Use artifact staging if running in VSTS - $destFolder = $env:BUILD_STAGINGDIRECTORY - } - - $BuildPackagePath = New-PSSignedBuildZip -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath -DestinationFolder $destFolder - Write-Verbose -Verbose "New-PSSignedBuildZip returned `$BuildPackagePath as: $BuildPackagePath" - Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$BuildPackagePath" - $buildPackageName = Split-Path -Path $BuildPackagePath -Leaf - $additionalFiles += $BuildPackagePath - } - - $psReleaseBranch = 'master' - $psReleaseFork = 'PowerShell' - $location = Join-Path -Path $PSScriptRoot -ChildPath 'PSRelease' - if(Test-Path $location) - { - Remove-Item -Path $location -Recurse -Force - } - - $gitBinFullPath = (Get-Command -Name git).Source - if (-not $gitBinFullPath) - { - throw "Git is required to proceed. Install from 'https://git-scm.com/download/win'" - } - - Write-Verbose "cloning -b $psReleaseBranch --quiet https://github.com/$psReleaseFork/PSRelease.git" -Verbose - & $gitBinFullPath clone -b $psReleaseBranch --quiet https://github.com/$psReleaseFork/PSRelease.git $location - - Push-Location -Path $PWD.Path - - $unresolvedRepoRoot = Join-Path -Path $PSScriptRoot '../..' - $resolvedRepoRoot = (Resolve-Path -Path $unresolvedRepoRoot).ProviderPath - - try - { - Write-Verbose "Starting build at $resolvedRepoRoot ..." -Verbose - Import-Module "$location/vstsBuild" -Force - Import-Module "$location/dockerBasedBuild" -Force - Clear-VstsTaskState - - $buildParameters = @{ - ReleaseTag = $ReleaseTag - BuildPackageName = $buildPackageName - } - - Invoke-Build -RepoPath $resolvedRepoRoot -BuildJsonPath './tools/releaseBuild/build.json' -Name $Name -Parameters $buildParameters -AdditionalFiles $AdditionalFiles - } - catch - { - Write-VstsError -Error $_ - } - finally{ - Write-VstsTaskState - exit 0 - } -} diff --git a/tools/releaseBuild/vstsbuild.sh b/tools/releaseBuild/vstsbuild.sh deleted file mode 100644 index d7d0363745f..00000000000 --- a/tools/releaseBuild/vstsbuild.sh +++ /dev/null @@ -1 +0,0 @@ -pwsh -command ".\vstsbuild.ps1 $*" diff --git a/tools/releaseTools.psm1 b/tools/releaseTools.psm1 index a50651d8889..6207c4be3f7 100644 --- a/tools/releaseTools.psm1 +++ b/tools/releaseTools.psm1 @@ -43,6 +43,7 @@ $Script:powershell_team = @( "Patrick Meinecke" "Steven Bucher" "PowerShell Team Bot" + "Justin Chung" ) # They are very active contributors, so we keep their email-login mappings here to save a few queries to Github. @@ -150,13 +151,20 @@ function Get-ChangeLog [Parameter(Mandatory = $true)] [string]$ThisReleaseTag, - [Parameter(Mandatory)] + [Parameter(Mandatory = $false)] [string]$Token, [Parameter()] [switch]$HasCherryPick ) + if(-not $Token) { + $Token = Get-GHDefaultAuthToken + if(-not $Token) { + throw "No GitHub Auth Token provided" + } + } + $tag_hash = git rev-parse "$LastReleaseTag^0" $format = '%H||%P||%aN||%aE||%s' $header = @{"Authorization"="token $Token"} @@ -360,6 +368,29 @@ function Get-ChangeLog Write-Output "[${version}]: https://github.com/PowerShell/PowerShell/compare/${LastReleaseTag}...${ThisReleaseTag}`n" } +function Get-GHDefaultAuthToken { + $IsGHCLIInstalled = $false + if (Get-command -CommandType Application -Name gh -ErrorAction SilentlyContinue) { + $IsGHCLIInstalled = $true + } else { + Write-Error -Message "GitHub CLI is not installed. Please install it from https://cli.github.com/" -ErrorAction Stop + } + + if ($IsGHCLIInstalled) { + try { + $Token = & gh auth token + } catch { + Write-Error -Message "Please login to GitHub CLI using 'gh auth login'" + } + } + + if (-not $Token) { + $Token = Read-Host -Prompt "Enter GitHub Auth Token" + } + + return $Token +} + function PrintChangeLog($clSection, $sectionTitle, [switch] $Compress) { if ($clSection.Count -gt 0) { "### $sectionTitle`n" diff --git a/tools/super-linter/config/super-linter.env b/tools/super-linter/config/super-linter.env new file mode 100644 index 00000000000..e7324b0feb9 --- /dev/null +++ b/tools/super-linter/config/super-linter.env @@ -0,0 +1,8 @@ +VALIDATE_ALL_CODEBASE=false +DEFAULT_BRANCH=master +FILTER_REGEX_INCLUDE=.*\.md +VALIDATE_EDITORCONFIG=false +VALIDATE_JSCPD=false +VALIDATE_CHECKOV=false +FIX_MARKDOWN_PRETTIER=true +FIX_MARKDOWN=true diff --git a/tools/super-linter/super-linter.ps1 b/tools/super-linter/super-linter.ps1 new file mode 100644 index 00000000000..571ba9c7f8d --- /dev/null +++ b/tools/super-linter/super-linter.ps1 @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +param( + [string]$RepoRoot = (Join-Path -Path $PSScriptRoot -ChildPath '../..'), + [string]$Platform +) + +$resolvedPath = (Resolve-Path $RepoRoot).ProviderPath +$platformParam = @() +if ($Platform) { + $platformParam = @("--platform", $Platform) +} + +docker run $platformParam -e RUN_LOCAL=true --env-file "$PSScriptRoot/config/super-linter.env" -v "${resolvedPath}:/tmp/lint" ghcr.io/super-linter/super-linter:latest