1297 lines
45 KiB
PowerShell
Executable File
1297 lines
45 KiB
PowerShell
Executable File
<#
|
||
.SYNOPSIS
|
||
Monitor recent Gitea Actions build runs and notify Discord about build status.
|
||
|
||
.DESCRIPTION
|
||
This script is intended to run after the main build pipeline. It queries a self-hosted Gitea instance,
|
||
analyzes recent Actions runs for one or more repositories, parses logs where available, caches processed
|
||
run IDs locally, and sends grouped Discord embed notifications.
|
||
|
||
Required token permissions depend on your Gitea version and repository visibility. For private repos,
|
||
use a personal access token that can read repository Actions/jobs and logs.
|
||
|
||
Sample Discord payload structure:
|
||
{
|
||
"username": "Gitea Build Monitor",
|
||
"embeds": [
|
||
{
|
||
"title": "CRITICAL: owner/repo build #123",
|
||
"url": "http://127.0.0.1:3000/owner/repo/actions/runs/123",
|
||
"color": 15158332,
|
||
"fields": [
|
||
{ "name": "Status", "value": "❌ failure", "inline": true },
|
||
{ "name": "Branch", "value": "main", "inline": true },
|
||
{ "name": "Commit", "value": "`abcdef12` Fix build", "inline": false }
|
||
]
|
||
}
|
||
]
|
||
}
|
||
#>
|
||
|
||
param(
|
||
[switch]$DryRun,
|
||
[switch]$ValidateOnly
|
||
)
|
||
|
||
Set-StrictMode -Version Latest
|
||
$ErrorActionPreference = 'Stop'
|
||
|
||
$GiteaApiToken = if (-not [string]::IsNullOrWhiteSpace($env:API_KEY)) { $env:API_KEY } else { $env:GITEA_API_TOKEN }
|
||
|
||
$Config = @{
|
||
GiteaBaseUrl = 'http://127.0.0.1:3000'
|
||
ApiToken = $GiteaApiToken
|
||
DiscordWebhookUrl = $env:DISCORD_WEBHOOK_URL
|
||
|
||
# Supports multiple repositories.
|
||
Repositories = @(
|
||
@{ Owner = 'zaine'; Name = 'org_web' }
|
||
)
|
||
|
||
LookbackMinutes = 30
|
||
SlowBuildThresholdMins = 20
|
||
CacheFile = Join-Path $PSScriptRoot 'gitea-build-monitor-cache.json'
|
||
LogFile = Join-Path $PSScriptRoot 'gitea-build-monitor.log'
|
||
BuildLogFile = Join-Path $PSScriptRoot 'org-web-build.log'
|
||
BuildSuccessPattern = 'JSON saved to output/test.json'
|
||
MaxJobsPerRepository = 50
|
||
MaxLogBytesToAnalyze = 250000
|
||
MaxDiscordEmbeds = 10
|
||
RetryCount = 3
|
||
RetryDelaySeconds = 2
|
||
AuthoringTestsEnabled = $true
|
||
AuthoringServiceDirectory = '/home/zaine/master-folder/org-platform/authoring-service'
|
||
AuthoringTestsPattern = 'test_authoring_server.py'
|
||
AuthoringTestsDirectory = '/home/zaine/master-folder/org-platform/authoring-service/tests'
|
||
AuthoringTestTimeoutSec = 120
|
||
|
||
# Tune these patterns to your build tooling.
|
||
ErrorPatterns = @(
|
||
'(?im)\b(error|exception|fatal|failed|failure)\b',
|
||
'(?im)\bbuild failed\b',
|
||
'(?im)\btest(s)? failed\b'
|
||
)
|
||
WarningPatterns = @(
|
||
'(?im)\bwarning\b',
|
||
'(?im)\bdeprecated\b'
|
||
)
|
||
|
||
# When true, completed successful runs are reported too.
|
||
NotifySuccessfulBuilds = $true
|
||
MaxPreviousFailures = 5
|
||
AlwaysNotifyBuildStatus = $true
|
||
|
||
# When true, successful but slow/warned jobs are reported too.
|
||
# Keep this false if Discord should only receive notifications for failed/cancelled/timed-out jobs.
|
||
NotifyWarningsAndSlow = $false
|
||
}
|
||
|
||
function Write-Log {
|
||
param(
|
||
[Parameter(Mandatory)]
|
||
[string]$Message,
|
||
|
||
[ValidateSet('INFO', 'WARN', 'ERROR')]
|
||
[string]$Level = 'INFO'
|
||
)
|
||
|
||
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('o'), $Level, $Message
|
||
Write-Host $line
|
||
Add-Content -Path $Config.LogFile -Value $line
|
||
}
|
||
|
||
function Assert-Config {
|
||
if ([string]::IsNullOrWhiteSpace($Config.ApiToken)) {
|
||
throw 'Missing Gitea API token. Set $Config.ApiToken or the GITEA_API_TOKEN environment variable.'
|
||
}
|
||
if ([string]::IsNullOrWhiteSpace($Config.DiscordWebhookUrl)) {
|
||
throw 'Missing Discord webhook URL. Set $Config.DiscordWebhookUrl or the DISCORD_WEBHOOK_URL environment variable.'
|
||
}
|
||
if (-not $Config.Repositories -or $Config.Repositories.Count -eq 0) {
|
||
throw 'Configure at least one repository in $Config.Repositories.'
|
||
}
|
||
}
|
||
|
||
function Join-Url {
|
||
param(
|
||
[Parameter(Mandatory)][string]$Base,
|
||
[Parameter(Mandatory)][string]$Path
|
||
)
|
||
|
||
'{0}/{1}' -f $Base.TrimEnd('/'), $Path.TrimStart('/')
|
||
}
|
||
|
||
function Invoke-GiteaApi {
|
||
param(
|
||
[Parameter(Mandatory)][string]$Path,
|
||
[ValidateSet('GET', 'POST')][string]$Method = 'GET',
|
||
[object]$Body = $null,
|
||
[switch]$RawText
|
||
)
|
||
|
||
$uri = Join-Url -Base $Config.GiteaBaseUrl -Path $Path
|
||
$headers = @{
|
||
Authorization = "token $($Config.ApiToken)"
|
||
Accept = if ($RawText) { 'text/plain' } else { 'application/json' }
|
||
}
|
||
|
||
for ($attempt = 1; $attempt -le [int]$Config.RetryCount; $attempt++) {
|
||
try {
|
||
$parameters = @{
|
||
Uri = $uri
|
||
Method = $Method
|
||
Headers = $headers
|
||
ErrorAction = 'Stop'
|
||
}
|
||
|
||
if ($null -ne $Body) {
|
||
$parameters.Body = ($Body | ConvertTo-Json -Depth 10)
|
||
$parameters.ContentType = 'application/json'
|
||
}
|
||
|
||
return Invoke-RestMethod @parameters
|
||
}
|
||
catch {
|
||
$status = $null
|
||
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
||
$status = [int]$_.Exception.Response.StatusCode
|
||
}
|
||
|
||
Write-Log -Level 'WARN' -Message ("Gitea API call failed, attempt {0}/{1}, status {2}: {3}" -f $attempt, $Config.RetryCount, $status, $_.Exception.Message)
|
||
if ($attempt -ge [int]$Config.RetryCount) {
|
||
throw
|
||
}
|
||
Start-Sleep -Seconds ([int]$Config.RetryDelaySeconds * $attempt)
|
||
}
|
||
}
|
||
}
|
||
|
||
function Invoke-DiscordWebhook {
|
||
param([Parameter(Mandatory)][hashtable]$Payload)
|
||
|
||
if ($DryRun) {
|
||
Write-Log -Message "Dry run enabled; Discord payload was not sent: $($Payload | ConvertTo-Json -Depth 12 -Compress)"
|
||
return
|
||
}
|
||
|
||
for ($attempt = 1; $attempt -le [int]$Config.RetryCount; $attempt++) {
|
||
try {
|
||
Invoke-RestMethod -Uri $Config.DiscordWebhookUrl -Method POST -ContentType 'application/json' -Body ($Payload | ConvertTo-Json -Depth 12) -ErrorAction Stop | Out-Null
|
||
return
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message ("Discord webhook failed, attempt {0}/{1}: {2}" -f $attempt, $Config.RetryCount, $_.Exception.Message)
|
||
if ($attempt -ge [int]$Config.RetryCount) {
|
||
throw
|
||
}
|
||
Start-Sleep -Seconds ([int]$Config.RetryDelaySeconds * $attempt)
|
||
}
|
||
}
|
||
}
|
||
|
||
function Read-ProcessedCache {
|
||
if (-not (Test-Path -Path $Config.CacheFile)) {
|
||
return @{}
|
||
}
|
||
|
||
try {
|
||
$json = Get-Content -Path $Config.CacheFile -Raw
|
||
if ([string]::IsNullOrWhiteSpace($json)) {
|
||
return @{}
|
||
}
|
||
|
||
$data = $json | ConvertFrom-Json
|
||
$cache = @{}
|
||
foreach ($property in $data.PSObject.Properties) {
|
||
$cache[$property.Name] = [datetime]$property.Value
|
||
}
|
||
return $cache
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not read cache file, starting with an empty cache: $($_.Exception.Message)"
|
||
return @{}
|
||
}
|
||
}
|
||
|
||
function Save-ProcessedCache {
|
||
param([Parameter(Mandatory)][hashtable]$Cache)
|
||
|
||
$cutoff = (Get-Date).AddDays(-14)
|
||
$trimmed = @{}
|
||
foreach ($key in $Cache.Keys) {
|
||
if ([datetime]$Cache[$key] -ge $cutoff) {
|
||
$trimmed[$key] = ([datetime]$Cache[$key]).ToString('o')
|
||
}
|
||
}
|
||
|
||
$directory = Split-Path -Parent $Config.CacheFile
|
||
if ($directory -and -not (Test-Path -Path $directory)) {
|
||
New-Item -ItemType Directory -Path $directory -Force | Out-Null
|
||
}
|
||
|
||
$trimmed | ConvertTo-Json -Depth 5 | Set-Content -Path $Config.CacheFile -Encoding UTF8
|
||
}
|
||
|
||
function Get-RepositoryOwner {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
return [string]$Repository['Owner']
|
||
}
|
||
|
||
function Get-RepositoryName {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
return [string]$Repository['Name']
|
||
}
|
||
|
||
function Get-RepositorySlug {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
return '{0}/{1}' -f (Get-RepositoryOwner -Repository $Repository), (Get-RepositoryName -Repository $Repository)
|
||
}
|
||
|
||
function Get-BuildRuns {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository
|
||
)
|
||
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$limit = [int]$Config.MaxJobsPerRepository
|
||
|
||
try {
|
||
$runsResponse = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/actions/runs?page=1&limit=$limit"
|
||
$runs = if ($runsResponse -is [array]) { $runsResponse } else { Get-ObjectValue -Object $runsResponse -Name 'workflow_runs' }
|
||
if (-not $runs) {
|
||
$runs = Get-ObjectValue -Object $runsResponse -Name 'runs'
|
||
}
|
||
if ($runs) {
|
||
foreach ($run in @($runs)) {
|
||
Add-Member -InputObject $run -NotePropertyName '_monitor_kind' -NotePropertyValue 'run' -Force
|
||
}
|
||
return @($runs)
|
||
}
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not fetch workflow runs for $(Get-RepositorySlug -Repository $Repository); falling back to jobs endpoint: $($_.Exception.Message)"
|
||
}
|
||
|
||
$jobsResponse = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/actions/jobs?page=1&limit=$limit"
|
||
$jobs = if ($jobsResponse -is [array]) { $jobsResponse } else { Get-ObjectValue -Object $jobsResponse -Name 'jobs' }
|
||
if ($jobs) {
|
||
foreach ($job in @($jobs)) {
|
||
Add-Member -InputObject $job -NotePropertyName '_monitor_kind' -NotePropertyValue 'job' -Force
|
||
}
|
||
return @($jobs)
|
||
}
|
||
|
||
return @()
|
||
}
|
||
|
||
function Get-CommitMessage {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][string]$Sha
|
||
)
|
||
|
||
if ([string]::IsNullOrWhiteSpace($Sha)) {
|
||
return ''
|
||
}
|
||
|
||
try {
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$encodedSha = [uri]::EscapeDataString($Sha)
|
||
$commit = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/git/commits/$encodedSha"
|
||
|
||
$message = Get-ObjectValue -Object $commit -Name 'message' -Default ''
|
||
if ($message) {
|
||
return (($message -split "`r?`n")[0]).Trim()
|
||
}
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not fetch commit message for $(Get-RepositorySlug -Repository $Repository) $($Sha): $($_.Exception.Message)"
|
||
}
|
||
|
||
return ''
|
||
}
|
||
|
||
function Get-FirstNonEmptyValue {
|
||
param([object[]]$Values)
|
||
|
||
foreach ($value in $Values) {
|
||
if (-not [string]::IsNullOrWhiteSpace([string]$value)) {
|
||
return [string]$value
|
||
}
|
||
}
|
||
|
||
return ''
|
||
}
|
||
|
||
function Get-CurrentBuildStatus {
|
||
$explicitStatus = Get-FirstNonEmptyValue -Values @(
|
||
$env:BUILD_STATUS,
|
||
$env:GITEA_BUILD_STATUS,
|
||
$env:CI_JOB_STATUS,
|
||
$env:JOB_STATUS
|
||
)
|
||
if ($explicitStatus) {
|
||
return $explicitStatus
|
||
}
|
||
|
||
$exitCode = Get-FirstNonEmptyValue -Values @(
|
||
$env:BUILD_EXIT_CODE,
|
||
$env:GITEA_BUILD_EXIT_CODE,
|
||
$env:CI_BUILD_EXIT_CODE
|
||
)
|
||
if ($exitCode) {
|
||
if ($exitCode -eq '0') {
|
||
return 'success'
|
||
}
|
||
return 'failure'
|
||
}
|
||
|
||
if ((Test-Path -Path $Config.BuildLogFile) -and -not [string]::IsNullOrWhiteSpace($Config.BuildSuccessPattern)) {
|
||
try {
|
||
$tail = Get-Content -Path $Config.BuildLogFile -Tail 25 -ErrorAction Stop
|
||
if (($tail -join "`n") -match [regex]::Escape($Config.BuildSuccessPattern)) {
|
||
return 'success'
|
||
}
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not inspect build log $($Config.BuildLogFile): $($_.Exception.Message)"
|
||
}
|
||
}
|
||
|
||
return 'success'
|
||
}
|
||
|
||
function New-CurrentBuildAnalysis {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
|
||
$repositorySlug = Get-RepositorySlug -Repository $Repository
|
||
$status = Get-CurrentBuildStatus
|
||
$severity = if (Test-IsSuccessfulStatus -Status $status) { 'Info' } else { 'Critical' }
|
||
$runId = Get-FirstNonEmptyValue -Values @(
|
||
$env:GITEA_RUN_ID,
|
||
$env:ACTIONS_RUN_ID,
|
||
$env:GITHUB_RUN_ID,
|
||
$env:CI_PIPELINE_ID,
|
||
$env:CI_RUN_ID
|
||
)
|
||
if (-not $runId) {
|
||
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddHHmmss')
|
||
}
|
||
|
||
$branch = Get-FirstNonEmptyValue -Values @(
|
||
$env:GITEA_REF_NAME,
|
||
$env:GITHUB_REF_NAME,
|
||
$env:CI_COMMIT_BRANCH,
|
||
$env:BRANCH_NAME
|
||
)
|
||
if (-not $branch) {
|
||
$branch = 'unknown'
|
||
}
|
||
|
||
$sha = Get-FirstNonEmptyValue -Values @(
|
||
$env:GITEA_SHA,
|
||
$env:GITHUB_SHA,
|
||
$env:CI_COMMIT_SHA
|
||
)
|
||
$buildUrl = Get-FirstNonEmptyValue -Values @(
|
||
$env:GITEA_RUN_URL,
|
||
$env:GITHUB_SERVER_URL
|
||
)
|
||
if (-not $buildUrl) {
|
||
$buildUrl = Join-Url -Base $Config.GiteaBaseUrl -Path "$repositorySlug/actions/runs/$runId"
|
||
}
|
||
|
||
return [pscustomobject]@{
|
||
Repository = $repositorySlug
|
||
Owner = Get-RepositoryOwner -Repository $Repository
|
||
Repo = Get-RepositoryName -Repository $Repository
|
||
JobId = 'current'
|
||
RunId = $runId
|
||
Branch = $branch
|
||
CommitSha = $sha
|
||
CommitMessage = ''
|
||
Status = $status
|
||
Duration = [timespan]::Zero
|
||
DurationText = 'current run'
|
||
BuildUrl = $buildUrl
|
||
ErrorSummary = if (Test-IsSuccessfulStatus -Status $status) { "Build completed successfully. Success marker: $($Config.BuildSuccessPattern)" } else { 'Build status reported as failed by the monitor environment.' }
|
||
HasErrors = -not (Test-IsSuccessfulStatus -Status $status)
|
||
HasWarnings = $false
|
||
IsSlow = $false
|
||
Severity = $severity
|
||
ShouldAlert = $true
|
||
StartedAt = Get-Date
|
||
IsCurrent = $true
|
||
}
|
||
}
|
||
|
||
function Get-JobLogText {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Job
|
||
)
|
||
|
||
try {
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$jobId = [uri]::EscapeDataString([string](Get-ObjectValue -Object $Job -Name 'id'))
|
||
$log = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/actions/jobs/$jobId/logs" -RawText
|
||
|
||
if ($null -eq $log) {
|
||
return ''
|
||
}
|
||
|
||
$text = [string]$log
|
||
if ($text.Length -gt [int]$Config.MaxLogBytesToAnalyze) {
|
||
return $text.Substring([Math]::Max(0, $text.Length - [int]$Config.MaxLogBytesToAnalyze))
|
||
}
|
||
return $text
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not fetch logs for job $(Get-ObjectValue -Object $Job -Name 'id'): $($_.Exception.Message)"
|
||
return ''
|
||
}
|
||
}
|
||
|
||
function Get-BuildLogText {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Build
|
||
)
|
||
|
||
$kind = [string](Get-ObjectValue -Object $Build -Name '_monitor_kind' -Default 'job')
|
||
if ($kind -ne 'run') {
|
||
return Get-JobLogText -Repository $Repository -Job $Build
|
||
}
|
||
|
||
try {
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$runId = [uri]::EscapeDataString([string](Get-ObjectValue -Object $Build -Name 'id'))
|
||
$response = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/actions/runs/$runId/jobs"
|
||
$jobs = Get-ObjectValue -Object $response -Name 'jobs'
|
||
if (-not $jobs) {
|
||
return ''
|
||
}
|
||
|
||
$logParts = New-Object System.Collections.Generic.List[string]
|
||
foreach ($job in @($jobs)) {
|
||
$conclusion = Get-ObjectValue -Object $job -Name 'conclusion'
|
||
$jobStatus = Get-ObjectValue -Object $job -Name 'status'
|
||
$status = if ($conclusion) { [string]$conclusion } elseif ($jobStatus) { [string]$jobStatus } else { 'unknown' }
|
||
if ($status.ToLowerInvariant() -notin @('failure', 'failed', 'cancelled', 'canceled', 'timed_out')) {
|
||
continue
|
||
}
|
||
|
||
$jobName = [string](Get-ObjectValue -Object $job -Name 'name' -Default (Get-ObjectValue -Object $job -Name 'id'))
|
||
$jobLog = Get-JobLogText -Repository $Repository -Job $job
|
||
if (-not [string]::IsNullOrWhiteSpace($jobLog)) {
|
||
$logParts.Add(("Job {0}:`n{1}" -f $jobName, $jobLog))
|
||
}
|
||
}
|
||
|
||
$text = ($logParts -join "`n`n")
|
||
if ($text.Length -gt [int]$Config.MaxLogBytesToAnalyze) {
|
||
return $text.Substring([Math]::Max(0, $text.Length - [int]$Config.MaxLogBytesToAnalyze))
|
||
}
|
||
return $text
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not fetch logs for run $(Get-ObjectValue -Object $Build -Name 'id'): $($_.Exception.Message)"
|
||
return ''
|
||
}
|
||
}
|
||
|
||
function Get-Duration {
|
||
param([Parameter(Mandatory)][object]$Job)
|
||
|
||
$start = $null
|
||
$end = $null
|
||
|
||
$startedAt = Get-ObjectValue -Object $Job -Name 'started_at'
|
||
$createdAt = Get-ObjectValue -Object $Job -Name 'created_at'
|
||
$completedAt = Get-ObjectValue -Object $Job -Name 'completed_at'
|
||
|
||
if ($startedAt) {
|
||
$start = [datetime]$startedAt
|
||
}
|
||
elseif ($createdAt) {
|
||
$start = [datetime]$createdAt
|
||
}
|
||
|
||
if ($completedAt) {
|
||
$end = [datetime]$completedAt
|
||
}
|
||
else {
|
||
$end = Get-Date
|
||
}
|
||
|
||
if ($null -eq $start) {
|
||
return [timespan]::Zero
|
||
}
|
||
|
||
return ($end.ToUniversalTime() - $start.ToUniversalTime())
|
||
}
|
||
|
||
function Get-ObjectValue {
|
||
param(
|
||
[Parameter(Mandatory)][object]$Object,
|
||
[Parameter(Mandatory)][string]$Name,
|
||
[object]$Default = $null
|
||
)
|
||
|
||
if ($null -eq $Object) {
|
||
return $Default
|
||
}
|
||
|
||
if ($Object -is [hashtable] -and $Object.ContainsKey($Name)) {
|
||
return $Object[$Name]
|
||
}
|
||
|
||
$property = $Object.PSObject.Properties[$Name]
|
||
if ($property) {
|
||
return $property.Value
|
||
}
|
||
|
||
return $Default
|
||
}
|
||
|
||
function Format-Duration {
|
||
param([Parameter(Mandatory)][timespan]$Duration)
|
||
|
||
if ($Duration.TotalHours -ge 1) {
|
||
return '{0}h {1}m {2}s' -f [int]$Duration.TotalHours, $Duration.Minutes, $Duration.Seconds
|
||
}
|
||
if ($Duration.TotalMinutes -ge 1) {
|
||
return '{0}m {1}s' -f [int]$Duration.TotalMinutes, $Duration.Seconds
|
||
}
|
||
return '{0}s' -f [int]$Duration.TotalSeconds
|
||
}
|
||
|
||
function Test-IsSuccessfulStatus {
|
||
param([Parameter(Mandatory)][string]$Status)
|
||
|
||
return $Status.ToLowerInvariant() -in @('success', 'succeeded', 'skipped', 'neutral')
|
||
}
|
||
|
||
function Test-IsCompletedStatus {
|
||
param([Parameter(Mandatory)][string]$Status)
|
||
|
||
return $Status.ToLowerInvariant() -in @('success', 'succeeded', 'skipped', 'neutral', 'failure', 'failed', 'cancelled', 'canceled', 'timed_out', 'completed')
|
||
}
|
||
|
||
function Get-LogSummary {
|
||
param(
|
||
[Parameter(Mandatory)][string]$LogText,
|
||
[switch]$IgnoreErrors
|
||
)
|
||
|
||
if ([string]::IsNullOrWhiteSpace($LogText)) {
|
||
return @{
|
||
HasErrors = $false
|
||
HasWarnings = $false
|
||
Summary = 'No log summary available.'
|
||
}
|
||
}
|
||
|
||
$matchedLines = New-Object System.Collections.Generic.List[string]
|
||
$hasErrors = $false
|
||
$hasWarnings = $false
|
||
|
||
foreach ($line in ($LogText -split "`r?`n")) {
|
||
$trimmed = $line.Trim()
|
||
if ([string]::IsNullOrWhiteSpace($trimmed)) {
|
||
continue
|
||
}
|
||
|
||
$isError = $false
|
||
if (-not $IgnoreErrors) {
|
||
foreach ($pattern in $Config.ErrorPatterns) {
|
||
if ($trimmed -match $pattern) {
|
||
$isError = $true
|
||
$hasErrors = $true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
$isWarning = $false
|
||
foreach ($pattern in $Config.WarningPatterns) {
|
||
if ($trimmed -match $pattern) {
|
||
$isWarning = $true
|
||
$hasWarnings = $true
|
||
break
|
||
}
|
||
}
|
||
|
||
if (($isError -or $isWarning) -and $matchedLines.Count -lt 8) {
|
||
$matchedLines.Add($trimmed)
|
||
}
|
||
}
|
||
|
||
$summary = if ($matchedLines.Count -gt 0) {
|
||
($matchedLines | Select-Object -First 8) -join "`n"
|
||
}
|
||
else {
|
||
'No matching warning or error lines found.'
|
||
}
|
||
|
||
return @{
|
||
HasErrors = $hasErrors
|
||
HasWarnings = $hasWarnings
|
||
Summary = $summary
|
||
}
|
||
}
|
||
|
||
function Get-Severity {
|
||
param(
|
||
[Parameter(Mandatory)][string]$Status,
|
||
[Parameter(Mandatory)][bool]$HasErrors,
|
||
[Parameter(Mandatory)][bool]$HasWarnings,
|
||
[Parameter(Mandatory)][bool]$IsSlow
|
||
)
|
||
|
||
$normalized = $Status.ToLowerInvariant()
|
||
if ($normalized -in @('failure', 'failed', 'cancelled', 'canceled', 'timed_out')) {
|
||
return 'Critical'
|
||
}
|
||
if ((-not (Test-IsSuccessfulStatus -Status $Status)) -and $HasErrors) {
|
||
return 'Critical'
|
||
}
|
||
if ($IsSlow -or $HasWarnings) {
|
||
return 'Warning'
|
||
}
|
||
return 'Info'
|
||
}
|
||
|
||
function Get-BuildStatus {
|
||
param([Parameter(Mandatory)][object]$Build)
|
||
|
||
$conclusion = Get-ObjectValue -Object $Build -Name 'conclusion'
|
||
if ([string]::IsNullOrWhiteSpace([string]$conclusion)) {
|
||
$conclusion = Get-ObjectValue -Object $Build -Name 'result'
|
||
}
|
||
if ([string]::IsNullOrWhiteSpace([string]$conclusion)) {
|
||
$conclusion = Get-ObjectValue -Object $Build -Name 'outcome'
|
||
}
|
||
if (-not [string]::IsNullOrWhiteSpace([string]$conclusion)) {
|
||
return [string]$conclusion
|
||
}
|
||
|
||
$status = Get-ObjectValue -Object $Build -Name 'status'
|
||
if (-not [string]::IsNullOrWhiteSpace([string]$status)) {
|
||
return [string]$status
|
||
}
|
||
|
||
return 'unknown'
|
||
}
|
||
|
||
function Analyze-Build {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Job
|
||
)
|
||
|
||
$status = Get-BuildStatus -Build $Job
|
||
$isSuccessfulStatus = Test-IsSuccessfulStatus -Status $status
|
||
$duration = Get-Duration -Job $Job
|
||
$isSlow = $duration.TotalMinutes -ge [double]$Config.SlowBuildThresholdMins
|
||
$logText = ''
|
||
|
||
$definitelyProblematicStatus = $status.ToLowerInvariant() -in @('failure', 'failed', 'cancelled', 'canceled', 'timed_out')
|
||
if ($definitelyProblematicStatus -or $Config.NotifyWarningsAndSlow) {
|
||
$logText = Get-BuildLogText -Repository $Repository -Build $Job
|
||
}
|
||
|
||
$logSummary = Get-LogSummary -LogText $logText -IgnoreErrors:$isSuccessfulStatus
|
||
$severity = Get-Severity -Status $status -HasErrors $logSummary.HasErrors -HasWarnings $logSummary.HasWarnings -IsSlow $isSlow
|
||
|
||
$shouldAlert = $severity -ne 'Info'
|
||
if (-not $Config.NotifyWarningsAndSlow -and $severity -eq 'Warning') {
|
||
$shouldAlert = $false
|
||
}
|
||
if (-not $shouldAlert -and $Config.NotifySuccessfulBuilds -and (Test-IsCompletedStatus -Status $status)) {
|
||
$shouldAlert = $true
|
||
}
|
||
|
||
$sha = [string](Get-ObjectValue -Object $Job -Name 'head_sha' -Default '')
|
||
$commitMessage = Get-CommitMessage -Repository $Repository -Sha $sha
|
||
$runId = [string](Get-ObjectValue -Object $Job -Name 'run_id' -Default (Get-ObjectValue -Object $Job -Name 'id'))
|
||
$htmlUrl = [string](Get-ObjectValue -Object $Job -Name 'html_url' -Default '')
|
||
$runUrl = [string](Get-ObjectValue -Object $Job -Name 'run_url' -Default '')
|
||
$repositorySlug = Get-RepositorySlug -Repository $Repository
|
||
$buildUrl = if ($htmlUrl) { $htmlUrl } elseif ($runUrl) { $runUrl } else { Join-Url -Base $Config.GiteaBaseUrl -Path "$repositorySlug/actions/runs/$runId" }
|
||
$startedAt = Get-ObjectValue -Object $Job -Name 'started_at'
|
||
$createdAt = Get-ObjectValue -Object $Job -Name 'created_at'
|
||
|
||
return [pscustomobject]@{
|
||
Repository = $repositorySlug
|
||
Owner = Get-RepositoryOwner -Repository $Repository
|
||
Repo = Get-RepositoryName -Repository $Repository
|
||
JobId = [string](Get-ObjectValue -Object $Job -Name 'id')
|
||
RunId = $runId
|
||
Branch = [string](Get-ObjectValue -Object $Job -Name 'head_branch' -Default 'unknown')
|
||
CommitSha = $sha
|
||
CommitMessage = $commitMessage
|
||
Status = $status
|
||
Duration = $duration
|
||
DurationText = Format-Duration -Duration $duration
|
||
BuildUrl = $buildUrl
|
||
ErrorSummary = $logSummary.Summary
|
||
HasErrors = $logSummary.HasErrors
|
||
HasWarnings = $logSummary.HasWarnings
|
||
IsSlow = $isSlow
|
||
Severity = $severity
|
||
ShouldAlert = $shouldAlert
|
||
StartedAt = if ($startedAt) { [datetime]$startedAt } elseif ($createdAt) { [datetime]$createdAt } else { Get-Date }
|
||
}
|
||
}
|
||
|
||
function New-DiscordEmbed {
|
||
param([Parameter(Mandatory)][object]$Analysis)
|
||
|
||
$isSuccess = Test-IsSuccessfulStatus -Status ([string]$Analysis.Status)
|
||
$emoji = if ($isSuccess) {
|
||
'✅'
|
||
}
|
||
else {
|
||
switch ($Analysis.Severity) {
|
||
'Critical' { '❌' }
|
||
'Warning' { '⚠️' }
|
||
default { 'ℹ️' }
|
||
}
|
||
}
|
||
|
||
$color = if ($isSuccess) {
|
||
3066993
|
||
}
|
||
else {
|
||
switch ($Analysis.Severity) {
|
||
'Critical' { 15158332 }
|
||
'Warning' { 16776960 }
|
||
default { 3447003 }
|
||
}
|
||
}
|
||
|
||
$shortSha = if ($Analysis.CommitSha.Length -ge 8) { $Analysis.CommitSha.Substring(0, 8) } else { $Analysis.CommitSha }
|
||
$commitValue = if ($shortSha) {
|
||
if ($Analysis.CommitMessage) { "``$shortSha`` $($Analysis.CommitMessage)" } else { "``$shortSha``" }
|
||
}
|
||
else {
|
||
'unknown'
|
||
}
|
||
|
||
$flags = @()
|
||
if ($Analysis.HasErrors) { $flags += 'errors found in logs' }
|
||
if ($Analysis.HasWarnings) { $flags += 'warnings found in logs' }
|
||
if ($Analysis.IsSlow) { $flags += "duration exceeded $($Config.SlowBuildThresholdMins)m" }
|
||
if ($flags.Count -eq 0) {
|
||
if ($isSuccess) {
|
||
$flags += 'build completed successfully'
|
||
}
|
||
else {
|
||
$flags += 'build status changed'
|
||
}
|
||
}
|
||
|
||
$summary = $Analysis.ErrorSummary
|
||
if ($summary.Length -gt 950) {
|
||
$summary = $summary.Substring(0, 947) + '...'
|
||
}
|
||
|
||
return @{
|
||
title = "$emoji $($Analysis.Status): $($Analysis.Repository) build #$($Analysis.RunId)"
|
||
url = $Analysis.BuildUrl
|
||
color = $color
|
||
description = ($flags -join ', ')
|
||
fields = @(
|
||
@{ name = 'Status'; value = "$emoji $($Analysis.Status)"; inline = $true },
|
||
@{ name = 'Branch'; value = $Analysis.Branch; inline = $true },
|
||
@{ name = 'Duration'; value = $Analysis.DurationText; inline = $true },
|
||
@{ name = 'Commit'; value = $commitValue; inline = $false },
|
||
@{ name = 'Summary'; value = $summary; inline = $false }
|
||
)
|
||
footer = @{
|
||
text = "Job ID $($Analysis.JobId) | Run ID $($Analysis.RunId)"
|
||
}
|
||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||
}
|
||
}
|
||
|
||
function Send-DiscordNotification {
|
||
param([Parameter(Mandatory)][object[]]$Analyses)
|
||
|
||
$alerting = @($Analyses | Where-Object { $_.ShouldAlert } | Sort-Object Severity, Repository, RunId)
|
||
if ($alerting.Count -eq 0) {
|
||
Write-Log -Message 'No alertable builds found.'
|
||
return
|
||
}
|
||
|
||
$maxEmbeds = [int]$Config.MaxDiscordEmbeds
|
||
if ($maxEmbeds -lt 1) {
|
||
$maxEmbeds = 1
|
||
}
|
||
for ($offset = 0; $offset -lt $alerting.Count; $offset += $maxEmbeds) {
|
||
$chunk = @($alerting | Select-Object -Skip $offset -First $maxEmbeds)
|
||
$criticalCount = @($chunk | Where-Object { $_.Severity -eq 'Critical' }).Count
|
||
$warningCount = @($chunk | Where-Object { $_.Severity -eq 'Warning' }).Count
|
||
$successCount = @($chunk | Where-Object { Test-IsSuccessfulStatus -Status ([string]$_.Status) }).Count
|
||
$completedCount = $chunk.Count
|
||
$embeds = @($chunk | ForEach-Object { New-DiscordEmbed -Analysis $_ })
|
||
$payload = @{
|
||
username = 'Gitea Build Monitor'
|
||
content = if ($criticalCount -gt 0) {
|
||
"❌ Gitea build monitor found $criticalCount failed build(s), $warningCount warning(s), and $successCount successful build(s)."
|
||
}
|
||
elseif ($warningCount -gt 0) {
|
||
"⚠️ Gitea build monitor found $warningCount warning(s) and $successCount successful build(s)."
|
||
}
|
||
elseif ($successCount -gt 0) {
|
||
"✅ Gitea build monitor found $successCount successful build(s)."
|
||
}
|
||
else {
|
||
"ℹ️ Gitea build monitor found $completedCount completed build(s)."
|
||
}
|
||
embeds = $embeds
|
||
}
|
||
|
||
Invoke-DiscordWebhook -Payload $payload
|
||
if ($DryRun) {
|
||
Write-Log -Message "Prepared Discord notification with $($chunk.Count) embed(s) in dry-run mode."
|
||
}
|
||
else {
|
||
Write-Log -Message "Sent Discord notification with $($chunk.Count) embed(s)."
|
||
}
|
||
}
|
||
}
|
||
|
||
function Get-BuildStatusEmoji {
|
||
param([Parameter(Mandatory)][string]$Status)
|
||
|
||
if (Test-IsSuccessfulStatus -Status $Status) {
|
||
return '✅'
|
||
}
|
||
|
||
$normalized = $Status.ToLowerInvariant()
|
||
if ($normalized -in @('failure', 'failed', 'cancelled', 'canceled', 'timed_out')) {
|
||
return '❌'
|
||
}
|
||
|
||
return 'ℹ️'
|
||
}
|
||
|
||
function Send-BuildStatusNotification {
|
||
param([Parameter(Mandatory)][object[]]$Analyses)
|
||
|
||
$latestByRepository = @($Analyses |
|
||
Group-Object Repository |
|
||
ForEach-Object {
|
||
$_.Group | Sort-Object StartedAt -Descending | Select-Object -First 1
|
||
} |
|
||
Sort-Object Repository)
|
||
|
||
if ($latestByRepository.Count -eq 0) {
|
||
Write-Log -Message "No completed builds found in the last $($Config.LookbackMinutes) minute(s)."
|
||
return
|
||
}
|
||
|
||
$embeds = New-Object System.Collections.Generic.List[object]
|
||
$failedLatestCount = 0
|
||
$successfulLatestCount = 0
|
||
$previousFailureCount = 0
|
||
|
||
foreach ($latest in $latestByRepository) {
|
||
$latestStatus = [string]$latest.Status
|
||
$emoji = Get-BuildStatusEmoji -Status $latestStatus
|
||
$isLatestSuccessful = Test-IsSuccessfulStatus -Status $latestStatus
|
||
if ($isLatestSuccessful) {
|
||
$successfulLatestCount++
|
||
}
|
||
elseif ($latest.Severity -eq 'Critical') {
|
||
$failedLatestCount++
|
||
}
|
||
|
||
$previousFailures = @($Analyses |
|
||
Where-Object {
|
||
$_.Repository -eq $latest.Repository -and
|
||
$_.RunId -ne $latest.RunId -and
|
||
$_.Severity -eq 'Critical'
|
||
} |
|
||
Sort-Object StartedAt -Descending |
|
||
Select-Object -First ([int]$Config.MaxPreviousFailures))
|
||
|
||
$previousFailureCount += $previousFailures.Count
|
||
$previousFailureText = if ($previousFailures.Count -gt 0) {
|
||
($previousFailures | ForEach-Object {
|
||
$shortSha = if ($_.CommitSha -and $_.CommitSha.Length -ge 8) { $_.CommitSha.Substring(0, 8) } else { $_.CommitSha }
|
||
$commitPart = if ($shortSha) { " ``$shortSha``" } else { '' }
|
||
"#{0} {1}{2}" -f $_.RunId, $_.Status, $commitPart
|
||
}) -join "`n"
|
||
}
|
||
else {
|
||
'None in this lookback window.'
|
||
}
|
||
|
||
if ($previousFailureText.Length -gt 950) {
|
||
$previousFailureText = $previousFailureText.Substring(0, 947) + '...'
|
||
}
|
||
|
||
$shortLatestSha = if ($latest.CommitSha -and $latest.CommitSha.Length -ge 8) { $latest.CommitSha.Substring(0, 8) } else { $latest.CommitSha }
|
||
$latestCommit = if ($shortLatestSha) {
|
||
if ($latest.CommitMessage) { "``$shortLatestSha`` $($latest.CommitMessage)" } else { "``$shortLatestSha``" }
|
||
}
|
||
else {
|
||
'unknown'
|
||
}
|
||
|
||
$color = if ($isLatestSuccessful) { 3066993 } elseif ($latest.Severity -eq 'Critical') { 15158332 } else { 3447003 }
|
||
$embeds.Add(@{
|
||
title = "$emoji Latest build: $($latest.Repository) #$($latest.RunId)"
|
||
url = $latest.BuildUrl
|
||
color = $color
|
||
description = if ($isLatestSuccessful) { 'Latest completed build is successful.' } elseif ($latest.Severity -eq 'Critical') { 'Latest completed build failed.' } else { 'Latest completed build has a non-success status.' }
|
||
fields = @(
|
||
@{ name = 'Latest status'; value = "$emoji $latestStatus"; inline = $true },
|
||
@{ name = 'Branch'; value = $latest.Branch; inline = $true },
|
||
@{ name = 'Duration'; value = $latest.DurationText; inline = $true },
|
||
@{ name = 'Commit'; value = $latestCommit; inline = $false },
|
||
@{ name = 'Previous failed builds'; value = $previousFailureText; inline = $false }
|
||
)
|
||
footer = @{
|
||
text = "Run ID $($latest.RunId)"
|
||
}
|
||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||
})
|
||
}
|
||
|
||
$content = if ($failedLatestCount -gt 0) {
|
||
"❌ Latest Gitea build status: $failedLatestCount failed, $successfulLatestCount successful. Previous failures in window: $previousFailureCount."
|
||
}
|
||
elseif ($successfulLatestCount -gt 0) {
|
||
"✅ Latest Gitea build status: $successfulLatestCount successful. Previous failures in window: $previousFailureCount."
|
||
}
|
||
else {
|
||
"ℹ️ Latest Gitea build status reported. Previous failures in window: $previousFailureCount."
|
||
}
|
||
|
||
$payload = @{
|
||
username = 'Gitea Build Monitor'
|
||
content = $content
|
||
embeds = @($embeds.ToArray())
|
||
}
|
||
|
||
Invoke-DiscordWebhook -Payload $payload
|
||
if ($DryRun) {
|
||
Write-Log -Message "Prepared build status notification with $($embeds.Count) embed(s) in dry-run mode."
|
||
}
|
||
else {
|
||
Write-Log -Message "Sent build status notification with $($embeds.Count) embed(s)."
|
||
}
|
||
}
|
||
|
||
function Get-PythonCommand {
|
||
$linuxVenvPython = Join-Path $Config.AuthoringServiceDirectory '.venv/bin/python'
|
||
$windowsVenvPython = Join-Path $Config.AuthoringServiceDirectory '.venv/Scripts/python.exe'
|
||
|
||
if (Test-Path -Path $linuxVenvPython) {
|
||
return $linuxVenvPython
|
||
}
|
||
if (Test-Path -Path $windowsVenvPython) {
|
||
return $windowsVenvPython
|
||
}
|
||
|
||
return 'python3'
|
||
}
|
||
|
||
function Invoke-AuthoringServerTests {
|
||
if (-not $Config.AuthoringTestsEnabled) {
|
||
return $null
|
||
}
|
||
|
||
if (-not (Test-Path -Path $Config.AuthoringTestsDirectory)) {
|
||
throw "Authoring test directory does not exist: $($Config.AuthoringTestsDirectory)"
|
||
}
|
||
|
||
$python = Get-PythonCommand
|
||
$arguments = @(
|
||
'-m',
|
||
'unittest',
|
||
'discover',
|
||
'-s',
|
||
$Config.AuthoringTestsDirectory,
|
||
'-p',
|
||
$Config.AuthoringTestsPattern,
|
||
'-v'
|
||
)
|
||
|
||
Write-Log -Message "Running authoring server tests with $python."
|
||
|
||
$process = New-Object System.Diagnostics.Process
|
||
$process.StartInfo.FileName = $python
|
||
foreach ($argument in $arguments) {
|
||
$process.StartInfo.ArgumentList.Add([string]$argument)
|
||
}
|
||
$process.StartInfo.WorkingDirectory = $Config.AuthoringServiceDirectory
|
||
$process.StartInfo.UseShellExecute = $false
|
||
$process.StartInfo.RedirectStandardOutput = $true
|
||
$process.StartInfo.RedirectStandardError = $true
|
||
|
||
[void]$process.Start()
|
||
$stdoutTask = $process.StandardOutput.ReadToEndAsync()
|
||
$stderrTask = $process.StandardError.ReadToEndAsync()
|
||
$completed = $process.WaitForExit([int]$Config.AuthoringTestTimeoutSec * 1000)
|
||
if (-not $completed) {
|
||
$process.Kill()
|
||
$process.WaitForExit()
|
||
}
|
||
else {
|
||
$process.WaitForExit()
|
||
}
|
||
|
||
$stdout = $stdoutTask.Result
|
||
$stderr = $stderrTask.Result
|
||
$output = (($stdout, $stderr) -join "`n").Trim()
|
||
$exitCode = if ($completed) { $process.ExitCode } else { 124 }
|
||
$ran = 0
|
||
$failures = 0
|
||
$errors = 0
|
||
$skipped = 0
|
||
|
||
if ($output -match 'Ran\s+(\d+)\s+tests?') {
|
||
$ran = [int]$Matches[1]
|
||
}
|
||
if ($output -match 'failures=(\d+)') {
|
||
$failures = [int]$Matches[1]
|
||
}
|
||
if ($output -match 'errors=(\d+)') {
|
||
$errors = [int]$Matches[1]
|
||
}
|
||
if ($output -match 'skipped=(\d+)') {
|
||
$skipped = [int]$Matches[1]
|
||
}
|
||
|
||
$status = if ($completed -and $exitCode -eq 0) { 'passed' } elseif (-not $completed) { 'timed out' } else { 'failed' }
|
||
$tailLines = @($output -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -Last 12)
|
||
$summary = if ($tailLines.Count -gt 0) { $tailLines -join "`n" } else { 'No test output captured.' }
|
||
if ($summary.Length -gt 950) {
|
||
$summary = $summary.Substring(0, 947) + '...'
|
||
}
|
||
|
||
$result = [pscustomobject]@{
|
||
Name = 'authoring_server.py tests'
|
||
Status = $status
|
||
Success = ($completed -and $exitCode -eq 0)
|
||
ExitCode = $exitCode
|
||
Ran = $ran
|
||
Failures = $failures
|
||
Errors = $errors
|
||
Skipped = $skipped
|
||
Summary = $summary
|
||
Completed = $completed
|
||
Timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||
}
|
||
|
||
Write-Log -Message ("Authoring server tests {0}: ran={1}, failures={2}, errors={3}, skipped={4}, exit={5}" -f $result.Status, $result.Ran, $result.Failures, $result.Errors, $result.Skipped, $result.ExitCode)
|
||
if (-not $result.Success) {
|
||
Write-Log -Level 'ERROR' -Message ("Authoring server test output:`n{0}" -f $result.Summary)
|
||
}
|
||
return $result
|
||
}
|
||
|
||
function Send-AuthoringTestNotification {
|
||
param([Parameter(Mandatory)][object]$Result)
|
||
|
||
$emoji = if ($Result.Success) { '✅' } else { '❌' }
|
||
$color = if ($Result.Success) { 3066993 } else { 15158332 }
|
||
$payload = @{
|
||
username = 'Gitea Build Monitor'
|
||
content = "$emoji Authoring server test result: $($Result.Status)."
|
||
embeds = @(
|
||
@{
|
||
title = "$emoji $($Result.Name)"
|
||
color = $color
|
||
description = "Unit test results from gitea-build-monitor.ps1"
|
||
fields = @(
|
||
@{ name = 'Status'; value = $Result.Status; inline = $true },
|
||
@{ name = 'Tests'; value = [string]$Result.Ran; inline = $true },
|
||
@{ name = 'Exit code'; value = [string]$Result.ExitCode; inline = $true },
|
||
@{ name = 'Failures'; value = [string]$Result.Failures; inline = $true },
|
||
@{ name = 'Errors'; value = [string]$Result.Errors; inline = $true },
|
||
@{ name = 'Skipped'; value = [string]$Result.Skipped; inline = $true },
|
||
@{ name = 'Summary'; value = $Result.Summary; inline = $false }
|
||
)
|
||
timestamp = $Result.Timestamp
|
||
}
|
||
)
|
||
}
|
||
|
||
Invoke-DiscordWebhook -Payload $payload
|
||
if ($DryRun) {
|
||
Write-Log -Message 'Prepared authoring server test notification in dry-run mode.'
|
||
}
|
||
else {
|
||
Write-Log -Message 'Sent authoring server test notification.'
|
||
}
|
||
}
|
||
|
||
function Test-IsRecentBuild {
|
||
param([Parameter(Mandatory)][object]$Job)
|
||
|
||
$cutoff = (Get-Date).ToUniversalTime().AddMinutes(-[int]$Config.LookbackMinutes)
|
||
$candidateDates = @(
|
||
Get-ObjectValue -Object $Job -Name 'started_at'
|
||
Get-ObjectValue -Object $Job -Name 'completed_at'
|
||
Get-ObjectValue -Object $Job -Name 'created_at'
|
||
) | Where-Object { $_ }
|
||
foreach ($date in $candidateDates) {
|
||
if (([datetime]$date).ToUniversalTime() -ge $cutoff) {
|
||
return $true
|
||
}
|
||
}
|
||
return $false
|
||
}
|
||
|
||
function Test-IsCompletedBuild {
|
||
param([Parameter(Mandatory)][object]$Build)
|
||
|
||
$conclusion = Get-ObjectValue -Object $Build -Name 'conclusion'
|
||
if ($conclusion) {
|
||
return Test-IsCompletedStatus -Status ([string]$conclusion)
|
||
}
|
||
|
||
$status = Get-ObjectValue -Object $Build -Name 'status'
|
||
if ($status) {
|
||
return Test-IsCompletedStatus -Status ([string]$status)
|
||
}
|
||
|
||
return $false
|
||
}
|
||
|
||
function Get-CacheKey {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Job
|
||
)
|
||
|
||
$kind = Get-ObjectValue -Object $Job -Name '_monitor_kind' -Default 'job'
|
||
$jobId = Get-ObjectValue -Object $Job -Name 'id'
|
||
$conclusion = Get-ObjectValue -Object $Job -Name 'conclusion'
|
||
$status = Get-ObjectValue -Object $Job -Name 'status' -Default 'unknown'
|
||
$state = if ($conclusion) { $conclusion } else { $status }
|
||
|
||
return "$(Get-RepositorySlug -Repository $Repository):$kind`:$jobId`:state:$state"
|
||
}
|
||
|
||
function Get-BuildSummaryCacheKey {
|
||
param([Parameter(Mandatory)][object[]]$Analyses)
|
||
|
||
$parts = @($Analyses |
|
||
Group-Object Repository |
|
||
ForEach-Object {
|
||
$latest = $_.Group | Sort-Object StartedAt -Descending | Select-Object -First 1
|
||
$previousFailureIds = @($_.Group |
|
||
Where-Object { $_.RunId -ne $latest.RunId -and $_.Severity -eq 'Critical' } |
|
||
Sort-Object StartedAt -Descending |
|
||
Select-Object -First ([int]$Config.MaxPreviousFailures) |
|
||
ForEach-Object { $_.RunId })
|
||
'{0}:latest:{1}:{2}:prevfail:{3}' -f $latest.Repository, $latest.RunId, $latest.Status, ($previousFailureIds -join ',')
|
||
} |
|
||
Sort-Object)
|
||
|
||
return 'build-summary:' + ($parts -join '|')
|
||
}
|
||
|
||
function Start-BuildMonitor {
|
||
Assert-Config
|
||
|
||
$logDirectory = Split-Path -Parent $Config.LogFile
|
||
if ($logDirectory -and -not (Test-Path -Path $logDirectory)) {
|
||
New-Item -ItemType Directory -Path $logDirectory -Force | Out-Null
|
||
}
|
||
|
||
if ($ValidateOnly) {
|
||
Write-Log -Message 'Validation mode completed: configuration and PowerShell runtime are valid.'
|
||
return
|
||
}
|
||
|
||
$cache = if ($DryRun) { @{} } else { Read-ProcessedCache }
|
||
$analyses = New-Object System.Collections.Generic.List[object]
|
||
|
||
foreach ($repository in $Config.Repositories) {
|
||
$currentAnalysis = New-CurrentBuildAnalysis -Repository $repository
|
||
$analyses.Add($currentAnalysis)
|
||
Write-Log -Message ("Prepared current build status for {0}: severity={1}, status={2}" -f $currentAnalysis.Repository, $currentAnalysis.Severity, $currentAnalysis.Status)
|
||
|
||
Write-Log -Message "Fetching recent Gitea Actions runs for $(Get-RepositorySlug -Repository $repository)."
|
||
|
||
try {
|
||
$jobs = Get-BuildRuns -Repository $repository
|
||
}
|
||
catch {
|
||
Write-Log -Level 'ERROR' -Message "Failed to retrieve builds for $(Get-RepositorySlug -Repository $repository): $($_.Exception.Message)"
|
||
continue
|
||
}
|
||
|
||
foreach ($job in $jobs) {
|
||
if (-not (Test-IsRecentBuild -Job $job)) {
|
||
continue
|
||
}
|
||
if (-not (Test-IsCompletedBuild -Build $job)) {
|
||
continue
|
||
}
|
||
|
||
try {
|
||
$analysis = Analyze-Build -Repository $repository -Job $job
|
||
$analyses.Add($analysis)
|
||
Write-Log -Message ("Analyzed job {0} for {1}: severity={2}, status={3}, duration={4}" -f (Get-ObjectValue -Object $job -Name 'id'), $analysis.Repository, $analysis.Severity, $analysis.Status, $analysis.DurationText)
|
||
}
|
||
catch {
|
||
Write-Log -Level 'ERROR' -Message "Failed to analyze job $(Get-ObjectValue -Object $job -Name 'id'): $($_.Exception.Message)"
|
||
}
|
||
}
|
||
}
|
||
|
||
if ($analyses.Count -gt 0) {
|
||
$summaryCacheKey = Get-BuildSummaryCacheKey -Analyses $analyses.ToArray()
|
||
if (-not $Config.AlwaysNotifyBuildStatus -and $cache.ContainsKey($summaryCacheKey)) {
|
||
Write-Log -Message 'Build status summary has already been sent.'
|
||
}
|
||
else {
|
||
Send-BuildStatusNotification -Analyses $analyses.ToArray()
|
||
if (-not $DryRun -and -not $Config.AlwaysNotifyBuildStatus) {
|
||
$cache[$summaryCacheKey] = Get-Date
|
||
}
|
||
}
|
||
}
|
||
else {
|
||
Write-Log -Message "No completed builds found in the last $($Config.LookbackMinutes) minute(s)."
|
||
}
|
||
|
||
$authoringTestResult = Invoke-AuthoringServerTests
|
||
if ($null -ne $authoringTestResult) {
|
||
Send-AuthoringTestNotification -Result $authoringTestResult
|
||
}
|
||
|
||
if (-not $DryRun) {
|
||
Save-ProcessedCache -Cache $cache
|
||
}
|
||
|
||
if ($null -ne $authoringTestResult -and -not $authoringTestResult.Success) {
|
||
throw 'Authoring server tests failed.'
|
||
}
|
||
}
|
||
|
||
try {
|
||
Start-BuildMonitor
|
||
}
|
||
catch {
|
||
Write-Log -Level 'ERROR' -Message "Build monitor failed: $($_.Exception.Message)"
|
||
Write-Log -Level 'ERROR' -Message "Exception type: $($_.Exception.GetType().FullName)"
|
||
if ($_.ScriptStackTrace) {
|
||
Write-Log -Level 'ERROR' -Message "Stack trace: $($_.ScriptStackTrace)"
|
||
}
|
||
exit 1
|
||
}
|