diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ab3116a508..cdbff2030f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -65,6 +65,22 @@ jobs: Build\Agent\TokenHygiene.Tests.ps1 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + # The native build runs this staleness check under Windows PowerShell 5.1; + # FwBuildHelpers.psm1 itself loads under either engine. + - name: Native object staleness tests (PowerShell 7) + id: native-obj-staleness-tests-pwsh + shell: pwsh + run: | + Build/Agent/NativeObjStaleness.Tests.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Native object staleness tests (Windows PowerShell 5.1) + id: native-obj-staleness-tests-winps + shell: powershell + run: | + Build\Agent\NativeObjStaleness.Tests.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + # -TokenHygiene fails CI on any hardcoded color/spacing literal in the Avalonia # surface (full-tree, no grandfathering -- unlike -CommentHygiene, which stays # advisory-only for humans here and only blocks agents locally). diff --git a/Build/Agent/FwBuildHelpers.psm1 b/Build/Agent/FwBuildHelpers.psm1 index f8a972115b..cf5bdd6e97 100644 --- a/Build/Agent/FwBuildHelpers.psm1 +++ b/Build/Agent/FwBuildHelpers.psm1 @@ -601,6 +601,148 @@ function Test-ViewsNativeArtifactsStale { return $latestSource -gt $oldestArtifact } +function Get-NativeMakefileModules { + <# + .SYNOPSIS + Returns the nmake-built native modules with the directories whose headers each one + compiles against. + .DESCRIPTION + Each entry names the module's folder under Obj\ and the repo-relative + directories, scanned without recursion, whose headers and makefiles feed its + objects. InputDirs covers the module makefile's UI include list, the folders its + sources reach through path-qualified includes, and the shared Include and Bld + folders. GeneratedDirs holds build-output folders as format strings taking the + configuration name; they exist only after a build. A directory missing from either + list leaves edits to its headers undetected for that module. + #> + $shared = @('Include', 'Include\unicode', 'Bld') + $generated = @('Output\{0}\Common') + return @( + [pscustomobject]@{ + Name = 'Views' + InputDirs = @('Src\views', 'Src\views\lib', 'Src\Generic', 'Src\AppCore', + 'Src\AppCore\Res', 'Src\Cellar', 'Src\DebugProcs', 'Src\Kernel', + 'Lib\src\graphite2\include\graphite2') + $shared + GeneratedDirs = $generated + }, + [pscustomobject]@{ + Name = 'FwKernel' + InputDirs = @('Src\Kernel', 'Src\Generic', 'Src\AppCore', 'Src\DebugProcs', + 'Src\Cellar') + $shared + GeneratedDirs = $generated + }, + [pscustomobject]@{ + Name = 'Generic' + InputDirs = @('Src\Generic') + $shared + GeneratedDirs = $generated + }, + [pscustomobject]@{ + Name = 'DebugProcs' + InputDirs = @('Src\DebugProcs') + $shared + GeneratedDirs = @() + } + ) +} + +function Remove-NativeObjDirsWithStaleInputs { + <# + .SYNOPSIS + Deletes a native module's intermediate folder when any header or makefile it builds + from is newer than its oldest object file. + .DESCRIPTION + The nmake rules in Bld rebuild an object only when its own source file changes, so a + header edit leaves every unedited file that includes it compiled against the old + header. When a class changes size that way, objects disagree about its layout and the + program corrupts memory at runtime. Deleting the module's Obj folder makes the next + native build compile all of it. + + The check is coarse by design: any newer input rebuilds the whole module, including + files that never include the changed header. + .PARAMETER Modules + Module entries shaped like Get-NativeMakefileModules output. Defaults to that list. + .OUTPUTS + The names of the modules whose folders were removed. + #> + param( + [Parameter(Mandatory)][string]$RepoRoot, + [Parameter(Mandatory)][string]$Configuration, + [object[]]$Modules = (Get-NativeMakefileModules) + ) + + $inputPatterns = @('*.h', '*.hpp', '*.inl', '*.idh', '*.mak') + # bldinc.h holds only version stamps that local builds regenerate daily; a stale copy in + # an object changes no layout. + $ignoredInputs = @('bldinc.h') + $removed = New-Object System.Collections.Generic.List[string] + + foreach ($module in $Modules) { + $objDir = Join-Path $RepoRoot (Join-Path "Obj\$Configuration" $module.Name) + if (-not (Test-Path -LiteralPath $objDir -PathType Container)) { + continue + } + + $oldestObj = $null + foreach ($obj in ([System.IO.DirectoryInfo]$objDir).EnumerateFiles('*.obj', [System.IO.SearchOption]::AllDirectories)) { + if ($null -eq $oldestObj -or $obj.LastWriteTimeUtc -lt $oldestObj.LastWriteTimeUtc) { + $oldestObj = $obj + } + } + if ($null -eq $oldestObj) { + continue + } + + $relativeDirs = @($module.InputDirs) + if ($module.PSObject.Properties['GeneratedDirs']) { + foreach ($format in $module.GeneratedDirs) { + $relativeDirs += ($format -f $Configuration) + } + } + + $newestInput = $null + foreach ($relativeDir in $relativeDirs) { + $inputDir = Join-Path $RepoRoot $relativeDir + if (-not (Test-Path -LiteralPath $inputDir -PathType Container)) { + continue + } + foreach ($pattern in $inputPatterns) { + foreach ($file in ([System.IO.DirectoryInfo]$inputDir).EnumerateFiles($pattern, [System.IO.SearchOption]::TopDirectoryOnly)) { + if ($ignoredInputs -contains $file.Name) { + continue + } + if ($null -eq $newestInput -or $file.LastWriteTimeUtc -gt $newestInput.LastWriteTimeUtc) { + $newestInput = $file + } + } + } + } + + if ($null -eq $newestInput -or $newestInput.LastWriteTimeUtc -le $oldestObj.LastWriteTimeUtc) { + continue + } + + $inputLabel = $newestInput.FullName.Substring($RepoRoot.TrimEnd('\').Length + 1) + $objLabel = $oldestObj.FullName.Substring($objDir.TrimEnd('\').Length + 1) + Write-Host ("[INFO] {0}: {1} ({2:yyyy-MM-dd HH:mm}) is newer than {3} ({4:yyyy-MM-dd HH:mm}); removing Obj\{5}\{0} so it rebuilds completely." -f ` + $module.Name, $inputLabel, $newestInput.LastWriteTime, $objLabel, $oldestObj.LastWriteTime, $Configuration) -ForegroundColor Yellow + $deleteErrors = @() + Remove-Item -LiteralPath $objDir -Recurse -Force -ErrorAction SilentlyContinue -ErrorVariable deleteErrors + if (Test-Path -LiteralPath $objDir) { + # The compiler's program-database server can hold a .pdb open between builds, and a + # leftover debug database does not affect what gets compiled. + $blocking = @(([System.IO.DirectoryInfo]$objDir).EnumerateFiles('*', [System.IO.SearchOption]::AllDirectories) | + Where-Object { $_.Extension -notin @('.pdb', '.idb') }) + if ($blocking.Count -gt 0) { + $reason = if ($deleteErrors.Count -gt 0) { $deleteErrors[0].Exception.Message } else { 'unknown error' } + throw "Could not remove $($blocking[0].FullName) while clearing stale native objects: $reason" + } + Write-Host "[WARN] $($module.Name): a debug database in Obj\$Configuration\$($module.Name) is in use and was left in place; every object was removed, so the module still rebuilds completely." -ForegroundColor Yellow + } + $removed.Add($module.Name) + } + + return ,$removed.ToArray() +} + function Set-TestAssertDialogEnvironment { <#! .SYNOPSIS @@ -765,6 +907,8 @@ Export-ModuleMember -Function @( 'Get-NewestWriteTimeUtc', 'Get-OldestWriteTimeUtc', 'Test-ViewsNativeArtifactsStale', + 'Get-NativeMakefileModules', + 'Remove-NativeObjDirsWithStaleInputs', 'Set-TestAssertDialogEnvironment', 'Disable-CrashDialog', 'Get-UnitppSummary', diff --git a/Build/Agent/NativeObjStaleness.Tests.ps1 b/Build/Agent/NativeObjStaleness.Tests.ps1 new file mode 100644 index 0000000000..fa7e87da21 --- /dev/null +++ b/Build/Agent/NativeObjStaleness.Tests.ps1 @@ -0,0 +1,221 @@ +<# +.SYNOPSIS + Fixture tests for Remove-NativeObjDirsWithStaleInputs in FwBuildHelpers.psm1. + +.DESCRIPTION + Builds a throwaway repo layout with object files and headers at chosen timestamps + and checks which module folders the helper removes. Run directly: + pwsh -File Build/Agent/NativeObjStaleness.Tests.ps1 +#> + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module (Join-Path $PSScriptRoot 'FwBuildHelpers.psm1') -Force + +$failures = New-Object System.Collections.ArrayList +$old = [datetime]::new(2026, 9, 2, 12, 0, 0, [System.DateTimeKind]::Utc) +$new = $old.AddDays(28) +$modules = @([pscustomobject]@{ + Name = 'Mod' + InputDirs = @('Src\mod', 'Include') + GeneratedDirs = @('Output\{0}\Common') +}) + +function New-Fixture { + $root = Join-Path ([System.IO.Path]::GetTempPath()) ("NativeObjFixture_" + [System.Guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $root | Out-Null + return $root +} + +function Set-FixtureFile { + param([string] $Root, [string] $RelativePath, [datetime] $WriteTimeUtc) + + $path = Join-Path $Root $RelativePath + New-Item -ItemType Directory -Path (Split-Path $path -Parent) -Force | Out-Null + Set-Content -LiteralPath $path -Value 'x' -Encoding ASCII + (Get-Item -LiteralPath $path).LastWriteTimeUtc = $WriteTimeUtc +} + +function Assert-Removal { + param([string] $Name, [scriptblock] $Arrange, [bool] $ExpectRemoved) + + $root = New-Fixture + try { + & $Arrange $root + $objDir = Join-Path $root 'Obj\Debug\Mod' + $existedBefore = Test-Path -LiteralPath $objDir + $removed = Remove-NativeObjDirsWithStaleInputs -RepoRoot $root -Configuration 'Debug' -Modules $modules 6>$null + $existsAfter = Test-Path -LiteralPath $objDir + $reportedRemoved = $removed -contains 'Mod' + if ($ExpectRemoved -and ($existsAfter -or -not $reportedRemoved)) { + [void]$script:failures.Add("FAIL [$Name]: expected Obj\Debug\Mod to be removed and reported") + } + if (-not $ExpectRemoved -and ($reportedRemoved -or ($existedBefore -and -not $existsAfter))) { + [void]$script:failures.Add("FAIL [$Name]: expected Obj\Debug\Mod to be left alone") + } + } + finally { + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue + } +} + +Assert-Removal 'header-newer-than-oldest-object' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\Unedited.obj' $old + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\Edited.obj' $new.AddMinutes(5) + Set-FixtureFile $r 'Src\mod\Layout.h' $new +} $true + +Assert-Removal 'shared-include-header-newer' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Include\Shared.h' $new +} $true + +Assert-Removal 'makefile-newer' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Src\mod\Mod.mak' $new +} $true + +Assert-Removal 'all-objects-newer-than-headers' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $new.AddMinutes(1) + Set-FixtureFile $r 'Obj\Debug\Mod\nopch\B.obj' $new.AddMinutes(2) + Set-FixtureFile $r 'Src\mod\Layout.h' $new +} $false + +# Source files are nmake's own job; only headers and makefiles count here. +Assert-Removal 'only-a-source-file-is-newer' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Src\mod\Layout.h' $old.AddMinutes(-1) + Set-FixtureFile $r 'Src\mod\Layout.cpp' $new +} $false + +# Test headers sit in a subfolder the module never includes from. +Assert-Removal 'header-in-unlisted-subfolder' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Src\mod\Test\TestLayout.h' $new +} $false + +Assert-Removal 'no-object-folder' { + param($r) + Set-FixtureFile $r 'Src\mod\Layout.h' $new +} $false + +Assert-Removal 'object-folder-without-objects' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\Mod.txt' $old + Set-FixtureFile $r 'Src\mod\Layout.h' $new +} $false + +Assert-Removal 'generated-header-newer' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Output\Debug\Common\CellarConstants.h' $new +} $true + +# Generated folders are per configuration; another configuration's output must not count. +Assert-Removal 'other-configuration-generated-header-newer' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Output\Release\Common\CellarConstants.h' $new +} $false + +Assert-Removal 'only-version-stamp-header-newer' { + param($r) + Set-FixtureFile $r 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $r 'Output\Debug\Common\bldinc.h' $new +} $false + +function Invoke-WithLockedFile { + param([string] $Name, [string] $LockedRelativePath, [bool] $ExpectThrow) + + $root = New-Fixture + $stream = $null + try { + Set-FixtureFile $root 'Obj\Debug\Mod\autopch\A.obj' $old + Set-FixtureFile $root 'Obj\Debug\Mod\autopch\B.obj' $old + Set-FixtureFile $root $LockedRelativePath $old + Set-FixtureFile $root 'Src\mod\Layout.h' $new + $stream = [System.IO.File]::Open((Join-Path $root $LockedRelativePath), 'Open', 'Read', 'None') + + $threw = $false + $removed = @() + try { + $removed = Remove-NativeObjDirsWithStaleInputs -RepoRoot $root -Configuration 'Debug' -Modules $modules 6>$null + } + catch { + $threw = $true + } + + $objDir = Join-Path $root 'Obj\Debug\Mod' + $objsLeft = @(Get-ChildItem -LiteralPath $objDir -Recurse -Filter '*.obj' -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -ne (Join-Path $root $LockedRelativePath) }) + if ($ExpectThrow -and -not $threw) { + [void]$script:failures.Add("FAIL [$Name]: expected a locked object file to fail the build") + } + if (-not $ExpectThrow -and ($threw -or -not ($removed -contains 'Mod') -or $objsLeft.Count -gt 0)) { + [void]$script:failures.Add("FAIL [$Name]: expected a locked debug database to be skipped with every object removed") + } + } + finally { + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue + } +} + +Invoke-WithLockedFile 'locked-debug-database-is-skipped' 'Obj\Debug\Mod\vc140.pdb' $false +Invoke-WithLockedFile 'locked-object-fails-the-build' 'Obj\Debug\Mod\autopch\Locked.obj' $true + +# Each real module's input folders must exist, or a typo would silently skip them. +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +foreach ($module in (Get-NativeMakefileModules)) { + foreach ($dir in $module.InputDirs) { + if (-not (Test-Path -LiteralPath (Join-Path $repoRoot $dir) -PathType Container)) { + [void]$failures.Add("FAIL [real-module-dirs]: $($module.Name) lists missing folder '$dir'") + } + } +} + +# A path-qualified include such as "../Cellar/FwXml.h" bypasses the makefile's include list, so +# the folder it reaches must be listed for every module that scans the including file. +foreach ($module in (Get-NativeMakefileModules)) { + $listed = @($module.InputDirs | ForEach-Object { [System.IO.Path]::GetFullPath((Join-Path $repoRoot $_)).TrimEnd('\') }) + foreach ($dir in $module.InputDirs) { + $fullDir = Join-Path $repoRoot $dir + foreach ($source in (Get-ChildItem -LiteralPath $fullDir -File -ErrorAction SilentlyContinue | + Where-Object { $_.Extension -in @('.cpp', '.c', '.h', '.hpp', '.inl') })) { + foreach ($match in (Select-String -LiteralPath $source.FullName -Pattern '^\s*#\s*include\s+"([^"]*[\\/][^"]*)"' -AllMatches)) { + $includePath = $match.Matches[0].Groups[1].Value + $target = Join-Path $source.DirectoryName $includePath + if (-not (Test-Path -LiteralPath $target -PathType Leaf)) { + # Leading ..\ segments may climb from a deeper include folder. + $target = Join-Path $repoRoot ($includePath -replace '^([.][.][\\/])+', '') + } + if (-not (Test-Path -LiteralPath $target -PathType Leaf)) { + continue + } + $targetDir = [System.IO.Path]::GetDirectoryName([System.IO.Path]::GetFullPath($target)).TrimEnd('\') + if ($listed -notcontains $targetDir) { + $relativeSource = $source.FullName.Substring($repoRoot.Length + 1) + [void]$failures.Add("FAIL [real-module-includes]: $($module.Name) scans $relativeSource, which includes a header in unlisted folder '$targetDir'") + } + } + } + } +} + +if ($failures.Count -gt 0) { + Write-Host '' + foreach ($f in $failures) { Write-Host $f -ForegroundColor Red } + Write-Host '' + Write-Host "$($failures.Count) test(s) failed." -ForegroundColor Red + exit 1 +} + +Write-Host 'All NativeObjStaleness tests passed.' -ForegroundColor Green +exit 0 diff --git a/Build/Agent/Remove-StaleNativeObjects.ps1 b/Build/Agent/Remove-StaleNativeObjects.ps1 new file mode 100644 index 0000000000..f6f17d5025 --- /dev/null +++ b/Build/Agent/Remove-StaleNativeObjects.ps1 @@ -0,0 +1,32 @@ +<# +.SYNOPSIS + Deletes the Obj folder of each nmake-built native module whose headers or makefiles + are newer than its oldest object, so the next native build compiles it completely. + +.DESCRIPTION + Wraps Remove-NativeObjDirsWithStaleInputs for MSBuild, which runs it after package + restore has copied its headers and before the first native module builds. Exits 1 + when a module's objects could not be removed. + +.EXAMPLE + Build/Agent/Remove-StaleNativeObjects.ps1 -RepoRoot . -Configuration Debug +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $RepoRoot, + [Parameter(Mandatory)][string] $Configuration +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module (Join-Path $PSScriptRoot 'FwBuildHelpers.psm1') -Force + +try { + [void](Remove-NativeObjDirsWithStaleInputs -RepoRoot (Resolve-Path $RepoRoot).Path -Configuration $Configuration) +} +catch { + Write-Host "[ERROR] $($_.Exception.Message)" -ForegroundColor Red + exit 1 +} +exit 0 diff --git a/Build/mkall.targets b/Build/mkall.targets index 46cfc58576..2d0a0cbfd1 100644 --- a/Build/mkall.targets +++ b/Build/mkall.targets @@ -69,7 +69,19 @@ ExcludedClsids="@(ExcludedClsids)" /> - + + + + + - - - - - + - + + $(dir-fwobj)\$(config-capital)\CellarConstants + $(CellarConstantsStagingDir)\CellarConstants.h + $(dir-fwoutputCommon)\CellarConstants.h + + +