param( [string]$OutputPath = 'D:\truf\runtime\freeze-diagnostics\runtime-lag.csv', [int]$IntervalSeconds = 30, [switch]$Once ) $ErrorActionPreference = 'Stop' throw 'Docker development copy: runtime control is disabled until container isolation is ready. See DOCKER_MIGRATION.md.' $created = $false $mutex = New-Object System.Threading.Mutex($true, 'Local\TrufRuntimeLagMonitor', [ref]$created) if (-not $created) { exit 0 } try { $counterTable = (Get-ItemProperty -LiteralPath 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Perflib\019').Counter $counterNames = @{} for ($index = 0; $index -lt $counterTable.Count; $index += 2) { $counterNames[[int]$counterTable[$index]] = $counterTable[$index + 1] } $counterPaths = @( "\$($counterNames[238])(_Total)\$($counterNames[6])", "\$($counterNames[4])\$($counterNames[822])", "\$($counterNames[234])(*)\$($counterNames[198])", "\$($counterNames[234])(*)\$($counterNames[206])", "\$($counterNames[234])(*)\$($counterNames[220])", "\$($counterNames[234])(*)\$($counterNames[222])" ) $gpuCounterPath = '\GPU Engine(*)\Utilization Percentage' try { Get-Counter -Counter $gpuCounterPath -MaxSamples 1 -ErrorAction Stop | Out-Null $counterPaths += $gpuCounterPath } catch { $gpuCounterPath = $null } $outputDirectory = Split-Path -Parent $OutputPath if (-not (Test-Path -LiteralPath $outputDirectory)) { New-Item -ItemType Directory -Path $outputDirectory | Out-Null } if (-not (Test-Path -LiteralPath $OutputPath)) { [IO.File]::AppendAllText( $OutputPath, "timestamp,cpu_percent,free_ram_mb,commit_used_mb,pagefile_used_mb,pages_input_sec,c_queue,c_latency_ms,d_queue,d_latency_ms,s_queue,s_latency_ms,h_queue,h_latency_ms,h_read_bytes_sec,h_write_bytes_sec,supervisor_pid,supervisor_private_mb,supervisor_handles,supervisor_threads,python_private_mb,python_handles,python_threads,largest_pid,largest_name,largest_private_mb,status_age_sec,scan_active,scan_limit,bundle_items,bundle_bytes,projection_items,projection_bytes,keycheck_items,keycheck_bytes,quarantine_items,quarantine_bytes,top_cpu_pid,top_cpu_name,top_cpu_percent,top_handles_pid,top_handles_name,top_handles_count,dpc_percent,interrupt_percent,interrupts_sec,gpu_engine_sum_percent,gpu_top_pid,gpu_top_name,gpu_top_engine_sum_percent`r`n", [Text.UTF8Encoding]::new($false) ) } # PhysicalDisk instances can omit drive letters, so resolve H: to its disk number once. $hDiskInstancePattern = '*h:*' try { foreach ($association in @(Get-CimInstance Win32_LogicalDiskToPartition -ErrorAction Stop)) { if ($association.Dependent.ToString().Contains('DeviceID = "H:"')) { $diskMatch = [regex]::Match($association.Antecedent.ToString(), 'Disk #(\d+)') if ($diskMatch.Success) { $hDiskInstancePattern = "$($diskMatch.Groups[1].Value)*" break } } } } catch { $hDiskInstancePattern = '*h:*' } $previousCpuByPid = @{} $previousProcessSampleAt = $null $logicalProcessorCount = [math]::Max(1, [Environment]::ProcessorCount) while ($true) { try { $sampleSets = @(Get-Counter -Counter $counterPaths -SampleInterval 1 -MaxSamples 2) $samples = $sampleSets[-1].CounterSamples $cpu = @($samples | Where-Object { $_.InstanceName -eq '_total' -and $_.Path.EndsWith("\$($counterNames[6])") })[0].CookedValue $pagesInput = @($samples | Where-Object { -not $_.InstanceName -and $_.Path.EndsWith("\$($counterNames[822])") })[0].CookedValue $cQueue = @($samples | Where-Object { $_.InstanceName -like '*c:*' -and $_.Path.EndsWith("\$($counterNames[198])") })[0].CookedValue $cLatency = @($samples | Where-Object { $_.InstanceName -like '*c:*' -and $_.Path.EndsWith("\$($counterNames[206])") })[0].CookedValue * 1000 $dQueue = @($samples | Where-Object { $_.InstanceName -like '*d:*' -and $_.Path.EndsWith("\$($counterNames[198])") })[0].CookedValue $dLatency = @($samples | Where-Object { $_.InstanceName -like '*d:*' -and $_.Path.EndsWith("\$($counterNames[206])") })[0].CookedValue * 1000 $sQueue = @($samples | Where-Object { $_.InstanceName -like '*s:*' -and $_.Path.EndsWith("\$($counterNames[198])") })[0].CookedValue $sLatency = @($samples | Where-Object { $_.InstanceName -like '*s:*' -and $_.Path.EndsWith("\$($counterNames[206])") })[0].CookedValue * 1000 $hDiskSamples = @($samples | Where-Object { $_.InstanceName -like $hDiskInstancePattern }) $hQueueSample = @($hDiskSamples | Where-Object { $_.CounterType.ToString() -eq 'NumberOfItems32' }) $hLatencySample = @($hDiskSamples | Where-Object { $_.CounterType.ToString() -eq 'AverageTimer32' }) $hThroughputSamples = @($hDiskSamples | Where-Object { $_.CounterType.ToString() -eq 'RateOfCountsPerSecond64' }) $hReadSample = @($hThroughputSamples | Select-Object -First 1) $hWriteSample = @($hThroughputSamples | Select-Object -Skip 1 -First 1) $hQueue = if ($hQueueSample.Count) { $hQueueSample[0].CookedValue } else { -1 } $hLatency = if ($hLatencySample.Count) { $hLatencySample[0].CookedValue * 1000 } else { -1 } $hReadBytes = if ($hReadSample.Count) { $hReadSample[0].CookedValue } else { -1 } $hWriteBytes = if ($hWriteSample.Count) { $hWriteSample[0].CookedValue } else { -1 } $os = Get-CimInstance Win32_OperatingSystem $pagefile = Get-CimInstance Win32_PageFileUsage | Where-Object { $_.Name -like 'D:*' } $python = @(Get-Process -Name python -ErrorAction SilentlyContinue) $allProcesses = @(Get-Process -ErrorAction SilentlyContinue) $largest = $allProcesses | Sort-Object PrivateMemorySize64 -Descending | Select-Object -First 1 $mostHandles = $allProcesses | Sort-Object HandleCount -Descending | Select-Object -First 1 $processSampleAt = Get-Date $elapsedMilliseconds = if ($previousProcessSampleAt) { ($processSampleAt - $previousProcessSampleAt).TotalMilliseconds } else { 0 } $currentCpuByPid = @{} $topCpu = $null $topCpuPercent = 0.0 foreach ($process in $allProcesses) { try { $processId = [int]$process.Id $totalProcessorMilliseconds = $process.TotalProcessorTime.TotalMilliseconds $currentCpuByPid[$processId] = $totalProcessorMilliseconds if ($elapsedMilliseconds -gt 0 -and $previousCpuByPid.ContainsKey($processId)) { $deltaMilliseconds = [math]::Max( 0, $totalProcessorMilliseconds - [double]$previousCpuByPid[$processId] ) $processCpuPercent = 100 * $deltaMilliseconds / ($elapsedMilliseconds * $logicalProcessorCount) if ($processCpuPercent -gt $topCpuPercent) { $topCpu = $process $topCpuPercent = $processCpuPercent } } } catch { continue } } $previousCpuByPid = $currentCpuByPid $previousProcessSampleAt = $processSampleAt $processorPerf = Get-CimInstance ` Win32_PerfFormattedData_PerfOS_Processor ` -Filter "Name='_Total'" ` -ErrorAction SilentlyContinue $gpuSamples = if ($gpuCounterPath) { @($samples | Where-Object { $_.Path -like '*\gpu engine(*)\utilization percentage' }) } else { @() } $gpuEngineSum = [double](($gpuSamples | Measure-Object CookedValue -Sum).Sum) $gpuByPid = @{} foreach ($sample in $gpuSamples) { if ($sample.InstanceName -match '^pid_(\d+)_') { $gpuProcessId = [int]$matches[1] if (-not $gpuByPid.ContainsKey($gpuProcessId)) { $gpuByPid[$gpuProcessId] = 0.0 } $gpuByPid[$gpuProcessId] += [double]$sample.CookedValue } } $gpuTop = $gpuByPid.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 1 $gpuTopProcess = if ($gpuTop) { $allProcesses | Where-Object { $_.Id -eq [int]$gpuTop.Key } | Select-Object -First 1 } else { $null } $supervisor = $null $instancePath = 'D:\truf\runtime\control\supervisor.instance.json' if (Test-Path -LiteralPath $instancePath) { $metadata = [IO.File]::ReadAllText($instancePath) | ConvertFrom-Json $supervisor = Get-Process -Id ([int]$metadata.pid) -ErrorAction SilentlyContinue } $statusPath = 'D:\truf\runtime\logs\supervisor.status.txt' $statusAge = if (Test-Path -LiteralPath $statusPath) { [math]::Max(0, ((Get-Date) - (Get-Item -LiteralPath $statusPath).LastWriteTime).TotalSeconds) } else { -1 } $statusText = if (Test-Path -LiteralPath $statusPath) { [IO.File]::ReadAllText($statusPath) } else { '' } $scanMatch = [regex]::Match($statusText, 'Scan workers: active=(\d+)/(\d+)') $pipelineMatch = [regex]::Match( $statusText, 'bundles=(\d+)/(\d+)B\s+projection=(\d+)/(\d+)B\s+candidates=(\d+)/(\d+)B\s+quarantine=(\d+)/(\d+)B' ) $values = @( (Get-Date).ToString('o'), [math]::Round($cpu, 2), [math]::Round($os.FreePhysicalMemory / 1KB, 1), [math]::Round(($os.TotalVirtualMemorySize - $os.FreeVirtualMemory) / 1KB, 1), [math]::Round((($pagefile | Measure-Object CurrentUsage -Sum).Sum), 1), [math]::Round($pagesInput, 2), [math]::Round($cQueue, 2), [math]::Round($cLatency, 2), [math]::Round($dQueue, 2), [math]::Round($dLatency, 2), [math]::Round($sQueue, 2), [math]::Round($sLatency, 2), [math]::Round($hQueue, 2), [math]::Round($hLatency, 2), [math]::Round($hReadBytes, 2), [math]::Round($hWriteBytes, 2), $(if ($supervisor) { $supervisor.Id } else { 0 }), $(if ($supervisor) { [math]::Round($supervisor.PrivateMemorySize64 / 1MB, 1) } else { 0 }), $(if ($supervisor) { $supervisor.HandleCount } else { 0 }), $(if ($supervisor) { $supervisor.Threads.Count } else { 0 }), [math]::Round((($python | Measure-Object PrivateMemorySize64 -Sum).Sum / 1MB), 1), [int](($python | Measure-Object HandleCount -Sum).Sum), [int](($python | ForEach-Object { $_.Threads.Count } | Measure-Object -Sum).Sum), $(if ($largest) { $largest.Id } else { 0 }), $(if ($largest) { ($largest.ProcessName -replace ',', '_') } else { '' }), $(if ($largest) { [math]::Round($largest.PrivateMemorySize64 / 1MB, 1) } else { 0 }), [math]::Round($statusAge, 1), $(if ($scanMatch.Success) { [int64]$scanMatch.Groups[1].Value } else { 0 }), $(if ($scanMatch.Success) { [int64]$scanMatch.Groups[2].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[1].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[2].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[3].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[4].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[5].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[6].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[7].Value } else { 0 }), $(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[8].Value } else { 0 }), $(if ($topCpu) { $topCpu.Id } else { 0 }), $(if ($topCpu) { ($topCpu.ProcessName -replace ',', '_') } else { '' }), [math]::Round($topCpuPercent, 2), $(if ($mostHandles) { $mostHandles.Id } else { 0 }), $(if ($mostHandles) { ($mostHandles.ProcessName -replace ',', '_') } else { '' }), $(if ($mostHandles) { $mostHandles.HandleCount } else { 0 }), $(if ($processorPerf) { [math]::Round($processorPerf.PercentDPCTime, 2) } else { 0 }), $(if ($processorPerf) { [math]::Round($processorPerf.PercentInterruptTime, 2) } else { 0 }), $(if ($processorPerf) { [math]::Round($processorPerf.InterruptsPersec, 2) } else { 0 }), [math]::Round($gpuEngineSum, 2), $(if ($gpuTop) { [int]$gpuTop.Key } else { 0 }), $(if ($gpuTopProcess) { ($gpuTopProcess.ProcessName -replace ',', '_') } else { '' }), $(if ($gpuTop) { [math]::Round([double]$gpuTop.Value, 2) } else { 0 }) ) $serializedValues = foreach ($value in $values) { if ($value -is [IFormattable]) { $value.ToString($null, [Globalization.CultureInfo]::InvariantCulture) } else { [string]$value } } [IO.File]::AppendAllText($OutputPath, (($serializedValues -join ',') + "`r`n"), [Text.UTF8Encoding]::new($false)) } catch { [IO.File]::AppendAllText( "$OutputPath.errors.log", "$(Get-Date -Format o) $($_.Exception.Message)`r`n", [Text.UTF8Encoding]::new($false) ) } if ($Once) { break } Start-Sleep -Seconds ([math]::Max(5, $IntervalSeconds)) } } finally { $mutex.ReleaseMutex() $mutex.Dispose() }