sashatrask
263 lines
14 KiB
PowerShell
263 lines
14 KiB
PowerShell
param(
|
|
[string]$OutputPath = 'D:\truf\runtime\freeze-diagnostics\runtime-lag.csv',
|
|
[int]$IntervalSeconds = 30,
|
|
[switch]$Once
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
throw 'Docker development copy: runtime control is disabled until container isolation is ready. See DOCKER_MIGRATION.md.'
|
|
$created = $false
|
|
$mutex = New-Object System.Threading.Mutex($true, 'Local\TrufRuntimeLagMonitor', [ref]$created)
|
|
if (-not $created) {
|
|
exit 0
|
|
}
|
|
|
|
try {
|
|
$counterTable = (Get-ItemProperty -LiteralPath 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Perflib\019').Counter
|
|
$counterNames = @{}
|
|
for ($index = 0; $index -lt $counterTable.Count; $index += 2) {
|
|
$counterNames[[int]$counterTable[$index]] = $counterTable[$index + 1]
|
|
}
|
|
$counterPaths = @(
|
|
"\$($counterNames[238])(_Total)\$($counterNames[6])",
|
|
"\$($counterNames[4])\$($counterNames[822])",
|
|
"\$($counterNames[234])(*)\$($counterNames[198])",
|
|
"\$($counterNames[234])(*)\$($counterNames[206])",
|
|
"\$($counterNames[234])(*)\$($counterNames[220])",
|
|
"\$($counterNames[234])(*)\$($counterNames[222])"
|
|
)
|
|
$gpuCounterPath = '\GPU Engine(*)\Utilization Percentage'
|
|
try {
|
|
Get-Counter -Counter $gpuCounterPath -MaxSamples 1 -ErrorAction Stop | Out-Null
|
|
$counterPaths += $gpuCounterPath
|
|
} catch {
|
|
$gpuCounterPath = $null
|
|
}
|
|
$outputDirectory = Split-Path -Parent $OutputPath
|
|
if (-not (Test-Path -LiteralPath $outputDirectory)) {
|
|
New-Item -ItemType Directory -Path $outputDirectory | Out-Null
|
|
}
|
|
if (-not (Test-Path -LiteralPath $OutputPath)) {
|
|
[IO.File]::AppendAllText(
|
|
$OutputPath,
|
|
"timestamp,cpu_percent,free_ram_mb,commit_used_mb,pagefile_used_mb,pages_input_sec,c_queue,c_latency_ms,d_queue,d_latency_ms,s_queue,s_latency_ms,h_queue,h_latency_ms,h_read_bytes_sec,h_write_bytes_sec,supervisor_pid,supervisor_private_mb,supervisor_handles,supervisor_threads,python_private_mb,python_handles,python_threads,largest_pid,largest_name,largest_private_mb,status_age_sec,scan_active,scan_limit,bundle_items,bundle_bytes,projection_items,projection_bytes,keycheck_items,keycheck_bytes,quarantine_items,quarantine_bytes,top_cpu_pid,top_cpu_name,top_cpu_percent,top_handles_pid,top_handles_name,top_handles_count,dpc_percent,interrupt_percent,interrupts_sec,gpu_engine_sum_percent,gpu_top_pid,gpu_top_name,gpu_top_engine_sum_percent`r`n",
|
|
[Text.UTF8Encoding]::new($false)
|
|
)
|
|
}
|
|
|
|
# PhysicalDisk instances can omit drive letters, so resolve H: to its disk number once.
|
|
$hDiskInstancePattern = '*h:*'
|
|
try {
|
|
foreach ($association in @(Get-CimInstance Win32_LogicalDiskToPartition -ErrorAction Stop)) {
|
|
if ($association.Dependent.ToString().Contains('DeviceID = "H:"')) {
|
|
$diskMatch = [regex]::Match($association.Antecedent.ToString(), 'Disk #(\d+)')
|
|
if ($diskMatch.Success) {
|
|
$hDiskInstancePattern = "$($diskMatch.Groups[1].Value)*"
|
|
break
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
$hDiskInstancePattern = '*h:*'
|
|
}
|
|
|
|
$previousCpuByPid = @{}
|
|
$previousProcessSampleAt = $null
|
|
$logicalProcessorCount = [math]::Max(1, [Environment]::ProcessorCount)
|
|
|
|
while ($true) {
|
|
try {
|
|
$sampleSets = @(Get-Counter -Counter $counterPaths -SampleInterval 1 -MaxSamples 2)
|
|
$samples = $sampleSets[-1].CounterSamples
|
|
$cpu = @($samples | Where-Object {
|
|
$_.InstanceName -eq '_total' -and $_.Path.EndsWith("\$($counterNames[6])")
|
|
})[0].CookedValue
|
|
$pagesInput = @($samples | Where-Object {
|
|
-not $_.InstanceName -and $_.Path.EndsWith("\$($counterNames[822])")
|
|
})[0].CookedValue
|
|
$cQueue = @($samples | Where-Object {
|
|
$_.InstanceName -like '*c:*' -and $_.Path.EndsWith("\$($counterNames[198])")
|
|
})[0].CookedValue
|
|
$cLatency = @($samples | Where-Object {
|
|
$_.InstanceName -like '*c:*' -and $_.Path.EndsWith("\$($counterNames[206])")
|
|
})[0].CookedValue * 1000
|
|
$dQueue = @($samples | Where-Object {
|
|
$_.InstanceName -like '*d:*' -and $_.Path.EndsWith("\$($counterNames[198])")
|
|
})[0].CookedValue
|
|
$dLatency = @($samples | Where-Object {
|
|
$_.InstanceName -like '*d:*' -and $_.Path.EndsWith("\$($counterNames[206])")
|
|
})[0].CookedValue * 1000
|
|
$sQueue = @($samples | Where-Object {
|
|
$_.InstanceName -like '*s:*' -and $_.Path.EndsWith("\$($counterNames[198])")
|
|
})[0].CookedValue
|
|
$sLatency = @($samples | Where-Object {
|
|
$_.InstanceName -like '*s:*' -and $_.Path.EndsWith("\$($counterNames[206])")
|
|
})[0].CookedValue * 1000
|
|
$hDiskSamples = @($samples | Where-Object { $_.InstanceName -like $hDiskInstancePattern })
|
|
$hQueueSample = @($hDiskSamples | Where-Object {
|
|
$_.CounterType.ToString() -eq 'NumberOfItems32'
|
|
})
|
|
$hLatencySample = @($hDiskSamples | Where-Object {
|
|
$_.CounterType.ToString() -eq 'AverageTimer32'
|
|
})
|
|
$hThroughputSamples = @($hDiskSamples | Where-Object {
|
|
$_.CounterType.ToString() -eq 'RateOfCountsPerSecond64'
|
|
})
|
|
$hReadSample = @($hThroughputSamples | Select-Object -First 1)
|
|
$hWriteSample = @($hThroughputSamples | Select-Object -Skip 1 -First 1)
|
|
$hQueue = if ($hQueueSample.Count) { $hQueueSample[0].CookedValue } else { -1 }
|
|
$hLatency = if ($hLatencySample.Count) { $hLatencySample[0].CookedValue * 1000 } else { -1 }
|
|
$hReadBytes = if ($hReadSample.Count) { $hReadSample[0].CookedValue } else { -1 }
|
|
$hWriteBytes = if ($hWriteSample.Count) { $hWriteSample[0].CookedValue } else { -1 }
|
|
$os = Get-CimInstance Win32_OperatingSystem
|
|
$pagefile = Get-CimInstance Win32_PageFileUsage | Where-Object { $_.Name -like 'D:*' }
|
|
$python = @(Get-Process -Name python -ErrorAction SilentlyContinue)
|
|
$allProcesses = @(Get-Process -ErrorAction SilentlyContinue)
|
|
$largest = $allProcesses | Sort-Object PrivateMemorySize64 -Descending | Select-Object -First 1
|
|
$mostHandles = $allProcesses | Sort-Object HandleCount -Descending | Select-Object -First 1
|
|
|
|
$processSampleAt = Get-Date
|
|
$elapsedMilliseconds = if ($previousProcessSampleAt) {
|
|
($processSampleAt - $previousProcessSampleAt).TotalMilliseconds
|
|
} else { 0 }
|
|
$currentCpuByPid = @{}
|
|
$topCpu = $null
|
|
$topCpuPercent = 0.0
|
|
foreach ($process in $allProcesses) {
|
|
try {
|
|
$processId = [int]$process.Id
|
|
$totalProcessorMilliseconds = $process.TotalProcessorTime.TotalMilliseconds
|
|
$currentCpuByPid[$processId] = $totalProcessorMilliseconds
|
|
if ($elapsedMilliseconds -gt 0 -and $previousCpuByPid.ContainsKey($processId)) {
|
|
$deltaMilliseconds = [math]::Max(
|
|
0,
|
|
$totalProcessorMilliseconds - [double]$previousCpuByPid[$processId]
|
|
)
|
|
$processCpuPercent = 100 * $deltaMilliseconds / ($elapsedMilliseconds * $logicalProcessorCount)
|
|
if ($processCpuPercent -gt $topCpuPercent) {
|
|
$topCpu = $process
|
|
$topCpuPercent = $processCpuPercent
|
|
}
|
|
}
|
|
} catch {
|
|
continue
|
|
}
|
|
}
|
|
$previousCpuByPid = $currentCpuByPid
|
|
$previousProcessSampleAt = $processSampleAt
|
|
|
|
$processorPerf = Get-CimInstance `
|
|
Win32_PerfFormattedData_PerfOS_Processor `
|
|
-Filter "Name='_Total'" `
|
|
-ErrorAction SilentlyContinue
|
|
$gpuSamples = if ($gpuCounterPath) {
|
|
@($samples | Where-Object { $_.Path -like '*\gpu engine(*)\utilization percentage' })
|
|
} else { @() }
|
|
$gpuEngineSum = [double](($gpuSamples | Measure-Object CookedValue -Sum).Sum)
|
|
$gpuByPid = @{}
|
|
foreach ($sample in $gpuSamples) {
|
|
if ($sample.InstanceName -match '^pid_(\d+)_') {
|
|
$gpuProcessId = [int]$matches[1]
|
|
if (-not $gpuByPid.ContainsKey($gpuProcessId)) {
|
|
$gpuByPid[$gpuProcessId] = 0.0
|
|
}
|
|
$gpuByPid[$gpuProcessId] += [double]$sample.CookedValue
|
|
}
|
|
}
|
|
$gpuTop = $gpuByPid.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 1
|
|
$gpuTopProcess = if ($gpuTop) {
|
|
$allProcesses | Where-Object { $_.Id -eq [int]$gpuTop.Key } | Select-Object -First 1
|
|
} else { $null }
|
|
$supervisor = $null
|
|
$instancePath = 'D:\truf\runtime\control\supervisor.instance.json'
|
|
if (Test-Path -LiteralPath $instancePath) {
|
|
$metadata = [IO.File]::ReadAllText($instancePath) | ConvertFrom-Json
|
|
$supervisor = Get-Process -Id ([int]$metadata.pid) -ErrorAction SilentlyContinue
|
|
}
|
|
$statusPath = 'D:\truf\runtime\logs\supervisor.status.txt'
|
|
$statusAge = if (Test-Path -LiteralPath $statusPath) {
|
|
[math]::Max(0, ((Get-Date) - (Get-Item -LiteralPath $statusPath).LastWriteTime).TotalSeconds)
|
|
} else { -1 }
|
|
$statusText = if (Test-Path -LiteralPath $statusPath) { [IO.File]::ReadAllText($statusPath) } else { '' }
|
|
$scanMatch = [regex]::Match($statusText, 'Scan workers: active=(\d+)/(\d+)')
|
|
$pipelineMatch = [regex]::Match(
|
|
$statusText,
|
|
'bundles=(\d+)/(\d+)B\s+projection=(\d+)/(\d+)B\s+candidates=(\d+)/(\d+)B\s+quarantine=(\d+)/(\d+)B'
|
|
)
|
|
$values = @(
|
|
(Get-Date).ToString('o'),
|
|
[math]::Round($cpu, 2),
|
|
[math]::Round($os.FreePhysicalMemory / 1KB, 1),
|
|
[math]::Round(($os.TotalVirtualMemorySize - $os.FreeVirtualMemory) / 1KB, 1),
|
|
[math]::Round((($pagefile | Measure-Object CurrentUsage -Sum).Sum), 1),
|
|
[math]::Round($pagesInput, 2),
|
|
[math]::Round($cQueue, 2),
|
|
[math]::Round($cLatency, 2),
|
|
[math]::Round($dQueue, 2),
|
|
[math]::Round($dLatency, 2),
|
|
[math]::Round($sQueue, 2),
|
|
[math]::Round($sLatency, 2),
|
|
[math]::Round($hQueue, 2),
|
|
[math]::Round($hLatency, 2),
|
|
[math]::Round($hReadBytes, 2),
|
|
[math]::Round($hWriteBytes, 2),
|
|
$(if ($supervisor) { $supervisor.Id } else { 0 }),
|
|
$(if ($supervisor) { [math]::Round($supervisor.PrivateMemorySize64 / 1MB, 1) } else { 0 }),
|
|
$(if ($supervisor) { $supervisor.HandleCount } else { 0 }),
|
|
$(if ($supervisor) { $supervisor.Threads.Count } else { 0 }),
|
|
[math]::Round((($python | Measure-Object PrivateMemorySize64 -Sum).Sum / 1MB), 1),
|
|
[int](($python | Measure-Object HandleCount -Sum).Sum),
|
|
[int](($python | ForEach-Object { $_.Threads.Count } | Measure-Object -Sum).Sum),
|
|
$(if ($largest) { $largest.Id } else { 0 }),
|
|
$(if ($largest) { ($largest.ProcessName -replace ',', '_') } else { '' }),
|
|
$(if ($largest) { [math]::Round($largest.PrivateMemorySize64 / 1MB, 1) } else { 0 }),
|
|
[math]::Round($statusAge, 1),
|
|
$(if ($scanMatch.Success) { [int64]$scanMatch.Groups[1].Value } else { 0 }),
|
|
$(if ($scanMatch.Success) { [int64]$scanMatch.Groups[2].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[1].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[2].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[3].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[4].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[5].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[6].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[7].Value } else { 0 }),
|
|
$(if ($pipelineMatch.Success) { [int64]$pipelineMatch.Groups[8].Value } else { 0 }),
|
|
$(if ($topCpu) { $topCpu.Id } else { 0 }),
|
|
$(if ($topCpu) { ($topCpu.ProcessName -replace ',', '_') } else { '' }),
|
|
[math]::Round($topCpuPercent, 2),
|
|
$(if ($mostHandles) { $mostHandles.Id } else { 0 }),
|
|
$(if ($mostHandles) { ($mostHandles.ProcessName -replace ',', '_') } else { '' }),
|
|
$(if ($mostHandles) { $mostHandles.HandleCount } else { 0 }),
|
|
$(if ($processorPerf) { [math]::Round($processorPerf.PercentDPCTime, 2) } else { 0 }),
|
|
$(if ($processorPerf) { [math]::Round($processorPerf.PercentInterruptTime, 2) } else { 0 }),
|
|
$(if ($processorPerf) { [math]::Round($processorPerf.InterruptsPersec, 2) } else { 0 }),
|
|
[math]::Round($gpuEngineSum, 2),
|
|
$(if ($gpuTop) { [int]$gpuTop.Key } else { 0 }),
|
|
$(if ($gpuTopProcess) { ($gpuTopProcess.ProcessName -replace ',', '_') } else { '' }),
|
|
$(if ($gpuTop) { [math]::Round([double]$gpuTop.Value, 2) } else { 0 })
|
|
)
|
|
$serializedValues = foreach ($value in $values) {
|
|
if ($value -is [IFormattable]) {
|
|
$value.ToString($null, [Globalization.CultureInfo]::InvariantCulture)
|
|
} else {
|
|
[string]$value
|
|
}
|
|
}
|
|
[IO.File]::AppendAllText($OutputPath, (($serializedValues -join ',') + "`r`n"), [Text.UTF8Encoding]::new($false))
|
|
} catch {
|
|
[IO.File]::AppendAllText(
|
|
"$OutputPath.errors.log",
|
|
"$(Get-Date -Format o) $($_.Exception.Message)`r`n",
|
|
[Text.UTF8Encoding]::new($false)
|
|
)
|
|
}
|
|
if ($Once) {
|
|
break
|
|
}
|
|
Start-Sleep -Seconds ([math]::Max(5, $IntervalSeconds))
|
|
}
|
|
} finally {
|
|
$mutex.ReleaseMutex()
|
|
$mutex.Dispose()
|
|
}
|