876 lines
29 KiB
PowerShell
Executable File
876 lines
29 KiB
PowerShell
Executable File
<#
|
||
.SYNOPSIS
|
||
Monitor recent Gitea Actions build jobs and notify Discord about failures, warnings, errors, or slow builds.
|
||
|
||
.DESCRIPTION
|
||
This script is intended to run after the main build pipeline. It queries a self-hosted Gitea instance,
|
||
analyzes recent Actions jobs for one or more repositories, parses logs where available, caches processed
|
||
job IDs locally, and sends grouped Discord embed notifications.
|
||
|
||
Required token permissions depend on your Gitea version and repository visibility. For private repos,
|
||
use a personal access token that can read repository Actions/jobs and logs.
|
||
|
||
Sample Discord payload structure:
|
||
{
|
||
"username": "Gitea Build Monitor",
|
||
"embeds": [
|
||
{
|
||
"title": "CRITICAL: owner/repo build #123",
|
||
"url": "http://127.0.0.1:3000/owner/repo/actions/runs/123",
|
||
"color": 15158332,
|
||
"fields": [
|
||
{ "name": "Status", "value": "❌ failure", "inline": true },
|
||
{ "name": "Branch", "value": "main", "inline": true },
|
||
{ "name": "Commit", "value": "`abcdef12` Fix build", "inline": false }
|
||
]
|
||
}
|
||
]
|
||
}
|
||
#>
|
||
|
||
param(
|
||
[switch]$DryRun,
|
||
[switch]$ValidateOnly
|
||
)
|
||
|
||
Set-StrictMode -Version Latest
|
||
$ErrorActionPreference = 'Stop'
|
||
|
||
# =========================
|
||
# Example configuration
|
||
# =========================
|
||
$Config = @{
|
||
GiteaBaseUrl = 'http://127.0.0.1:3000'
|
||
ApiToken = '328ae2196703fc988e9c21962341ba88709511ad'
|
||
DiscordWebhookUrl = 'https://discord.com/api/webhooks/1501605890581856430/j3zeMhCW-V583awzqhyrZMfVwzZl-RRCLdIa32jUiymbodzpu9WZaX_p3HiiQMtUUO0m'
|
||
|
||
# Supports multiple repositories.
|
||
Repositories = @(
|
||
@{ Owner = 'zaine'; Name = 'org_web' }
|
||
)
|
||
|
||
LookbackMinutes = 30
|
||
SlowBuildThresholdMins = 20
|
||
CacheFile = Join-Path $PSScriptRoot 'gitea-build-monitor-cache.json'
|
||
LogFile = Join-Path $PSScriptRoot 'gitea-build-monitor.log'
|
||
MaxJobsPerRepository = 50
|
||
MaxLogBytesToAnalyze = 250000
|
||
MaxDiscordEmbeds = 10
|
||
RetryCount = 3
|
||
RetryDelaySeconds = 2
|
||
AuthoringTestsEnabled = $true
|
||
AuthoringTestsPattern = 'test_authoring_server.py'
|
||
AuthoringTestsDirectory = Join-Path $PSScriptRoot 'tests'
|
||
AuthoringTestTimeoutSec = 120
|
||
|
||
# Tune these patterns to your build tooling.
|
||
ErrorPatterns = @(
|
||
'(?im)\b(error|exception|fatal|failed|failure)\b',
|
||
'(?im)\bbuild failed\b',
|
||
'(?im)\btest(s)? failed\b'
|
||
)
|
||
WarningPatterns = @(
|
||
'(?im)\bwarning\b',
|
||
'(?im)\bdeprecated\b'
|
||
)
|
||
|
||
# When true, successful but slow/warned jobs are reported too.
|
||
# Keep this false if Discord should only receive notifications for failed/cancelled/timed-out jobs.
|
||
NotifyWarningsAndSlow = $false
|
||
}
|
||
|
||
function Write-Log {
|
||
param(
|
||
[Parameter(Mandatory)]
|
||
[string]$Message,
|
||
|
||
[ValidateSet('INFO', 'WARN', 'ERROR')]
|
||
[string]$Level = 'INFO'
|
||
)
|
||
|
||
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('o'), $Level, $Message
|
||
Write-Host $line
|
||
Add-Content -Path $Config.LogFile -Value $line
|
||
}
|
||
|
||
function Assert-Config {
|
||
if ([string]::IsNullOrWhiteSpace($Config.ApiToken)) {
|
||
throw 'Missing Gitea API token. Set $Config.ApiToken or the GITEA_API_TOKEN environment variable.'
|
||
}
|
||
if ([string]::IsNullOrWhiteSpace($Config.DiscordWebhookUrl)) {
|
||
throw 'Missing Discord webhook URL. Set $Config.DiscordWebhookUrl or the DISCORD_WEBHOOK_URL environment variable.'
|
||
}
|
||
if (-not $Config.Repositories -or $Config.Repositories.Count -eq 0) {
|
||
throw 'Configure at least one repository in $Config.Repositories.'
|
||
}
|
||
}
|
||
|
||
function Join-Url {
|
||
param(
|
||
[Parameter(Mandatory)][string]$Base,
|
||
[Parameter(Mandatory)][string]$Path
|
||
)
|
||
|
||
'{0}/{1}' -f $Base.TrimEnd('/'), $Path.TrimStart('/')
|
||
}
|
||
|
||
function Invoke-GiteaApi {
|
||
param(
|
||
[Parameter(Mandatory)][string]$Path,
|
||
[ValidateSet('GET', 'POST')][string]$Method = 'GET',
|
||
[object]$Body = $null,
|
||
[switch]$RawText
|
||
)
|
||
|
||
$uri = Join-Url -Base $Config.GiteaBaseUrl -Path $Path
|
||
$headers = @{
|
||
Authorization = "token $($Config.ApiToken)"
|
||
Accept = if ($RawText) { 'text/plain' } else { 'application/json' }
|
||
}
|
||
|
||
for ($attempt = 1; $attempt -le [int]$Config.RetryCount; $attempt++) {
|
||
try {
|
||
$parameters = @{
|
||
Uri = $uri
|
||
Method = $Method
|
||
Headers = $headers
|
||
ErrorAction = 'Stop'
|
||
}
|
||
|
||
if ($null -ne $Body) {
|
||
$parameters.Body = ($Body | ConvertTo-Json -Depth 10)
|
||
$parameters.ContentType = 'application/json'
|
||
}
|
||
|
||
return Invoke-RestMethod @parameters
|
||
}
|
||
catch {
|
||
$status = $null
|
||
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
||
$status = [int]$_.Exception.Response.StatusCode
|
||
}
|
||
|
||
Write-Log -Level 'WARN' -Message ("Gitea API call failed, attempt {0}/{1}, status {2}: {3}" -f $attempt, $Config.RetryCount, $status, $_.Exception.Message)
|
||
if ($attempt -ge [int]$Config.RetryCount) {
|
||
throw
|
||
}
|
||
Start-Sleep -Seconds ([int]$Config.RetryDelaySeconds * $attempt)
|
||
}
|
||
}
|
||
}
|
||
|
||
function Invoke-DiscordWebhook {
|
||
param([Parameter(Mandatory)][hashtable]$Payload)
|
||
|
||
if ($DryRun) {
|
||
Write-Log -Message "Dry run enabled; Discord payload was not sent: $($Payload | ConvertTo-Json -Depth 12 -Compress)"
|
||
return
|
||
}
|
||
|
||
for ($attempt = 1; $attempt -le [int]$Config.RetryCount; $attempt++) {
|
||
try {
|
||
Invoke-RestMethod -Uri $Config.DiscordWebhookUrl -Method POST -ContentType 'application/json' -Body ($Payload | ConvertTo-Json -Depth 12) -ErrorAction Stop | Out-Null
|
||
return
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message ("Discord webhook failed, attempt {0}/{1}: {2}" -f $attempt, $Config.RetryCount, $_.Exception.Message)
|
||
if ($attempt -ge [int]$Config.RetryCount) {
|
||
throw
|
||
}
|
||
Start-Sleep -Seconds ([int]$Config.RetryDelaySeconds * $attempt)
|
||
}
|
||
}
|
||
}
|
||
|
||
function Read-ProcessedCache {
|
||
if (-not (Test-Path -Path $Config.CacheFile)) {
|
||
return @{}
|
||
}
|
||
|
||
try {
|
||
$json = Get-Content -Path $Config.CacheFile -Raw
|
||
if ([string]::IsNullOrWhiteSpace($json)) {
|
||
return @{}
|
||
}
|
||
|
||
$data = $json | ConvertFrom-Json
|
||
$cache = @{}
|
||
foreach ($property in $data.PSObject.Properties) {
|
||
$cache[$property.Name] = [datetime]$property.Value
|
||
}
|
||
return $cache
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not read cache file, starting with an empty cache: $($_.Exception.Message)"
|
||
return @{}
|
||
}
|
||
}
|
||
|
||
function Save-ProcessedCache {
|
||
param([Parameter(Mandatory)][hashtable]$Cache)
|
||
|
||
$cutoff = (Get-Date).AddDays(-14)
|
||
$trimmed = @{}
|
||
foreach ($key in $Cache.Keys) {
|
||
if ([datetime]$Cache[$key] -ge $cutoff) {
|
||
$trimmed[$key] = ([datetime]$Cache[$key]).ToString('o')
|
||
}
|
||
}
|
||
|
||
$directory = Split-Path -Parent $Config.CacheFile
|
||
if ($directory -and -not (Test-Path -Path $directory)) {
|
||
New-Item -ItemType Directory -Path $directory -Force | Out-Null
|
||
}
|
||
|
||
$trimmed | ConvertTo-Json -Depth 5 | Set-Content -Path $Config.CacheFile -Encoding UTF8
|
||
}
|
||
|
||
function Get-RepositoryOwner {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
return [string]$Repository['Owner']
|
||
}
|
||
|
||
function Get-RepositoryName {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
return [string]$Repository['Name']
|
||
}
|
||
|
||
function Get-RepositorySlug {
|
||
param([Parameter(Mandatory)][hashtable]$Repository)
|
||
return '{0}/{1}' -f (Get-RepositoryOwner -Repository $Repository), (Get-RepositoryName -Repository $Repository)
|
||
}
|
||
|
||
function Get-BuildRuns {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository
|
||
)
|
||
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$path = "/api/v1/repos/$owner/$repo/actions/jobs?page=1&limit=$($Config.MaxJobsPerRepository)"
|
||
$response = Invoke-GiteaApi -Path $path
|
||
|
||
$jobs = Get-ObjectValue -Object $response -Name 'jobs'
|
||
if ($jobs) {
|
||
return @($jobs)
|
||
}
|
||
|
||
return @()
|
||
}
|
||
|
||
function Get-CommitMessage {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][string]$Sha
|
||
)
|
||
|
||
if ([string]::IsNullOrWhiteSpace($Sha)) {
|
||
return ''
|
||
}
|
||
|
||
try {
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$encodedSha = [uri]::EscapeDataString($Sha)
|
||
$commit = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/git/commits/$encodedSha"
|
||
|
||
$message = Get-ObjectValue -Object $commit -Name 'message' -Default ''
|
||
if ($message) {
|
||
return (($message -split "`r?`n")[0]).Trim()
|
||
}
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not fetch commit message for $(Get-RepositorySlug -Repository $Repository) $($Sha): $($_.Exception.Message)"
|
||
}
|
||
|
||
return ''
|
||
}
|
||
|
||
function Get-JobLogText {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Job
|
||
)
|
||
|
||
try {
|
||
$owner = [uri]::EscapeDataString((Get-RepositoryOwner -Repository $Repository))
|
||
$repo = [uri]::EscapeDataString((Get-RepositoryName -Repository $Repository))
|
||
$jobId = [uri]::EscapeDataString([string](Get-ObjectValue -Object $Job -Name 'id'))
|
||
$log = Invoke-GiteaApi -Path "/api/v1/repos/$owner/$repo/actions/jobs/$jobId/logs" -RawText
|
||
|
||
if ($null -eq $log) {
|
||
return ''
|
||
}
|
||
|
||
$text = [string]$log
|
||
if ($text.Length -gt [int]$Config.MaxLogBytesToAnalyze) {
|
||
return $text.Substring([Math]::Max(0, $text.Length - [int]$Config.MaxLogBytesToAnalyze))
|
||
}
|
||
return $text
|
||
}
|
||
catch {
|
||
Write-Log -Level 'WARN' -Message "Could not fetch logs for job $(Get-ObjectValue -Object $Job -Name 'id'): $($_.Exception.Message)"
|
||
return ''
|
||
}
|
||
}
|
||
|
||
function Get-Duration {
|
||
param([Parameter(Mandatory)][object]$Job)
|
||
|
||
$start = $null
|
||
$end = $null
|
||
|
||
$startedAt = Get-ObjectValue -Object $Job -Name 'started_at'
|
||
$createdAt = Get-ObjectValue -Object $Job -Name 'created_at'
|
||
$completedAt = Get-ObjectValue -Object $Job -Name 'completed_at'
|
||
|
||
if ($startedAt) {
|
||
$start = [datetime]$startedAt
|
||
}
|
||
elseif ($createdAt) {
|
||
$start = [datetime]$createdAt
|
||
}
|
||
|
||
if ($completedAt) {
|
||
$end = [datetime]$completedAt
|
||
}
|
||
else {
|
||
$end = Get-Date
|
||
}
|
||
|
||
if ($null -eq $start) {
|
||
return [timespan]::Zero
|
||
}
|
||
|
||
return ($end.ToUniversalTime() - $start.ToUniversalTime())
|
||
}
|
||
|
||
function Get-ObjectValue {
|
||
param(
|
||
[Parameter(Mandatory)][object]$Object,
|
||
[Parameter(Mandatory)][string]$Name,
|
||
[object]$Default = $null
|
||
)
|
||
|
||
if ($null -eq $Object) {
|
||
return $Default
|
||
}
|
||
|
||
if ($Object -is [hashtable] -and $Object.ContainsKey($Name)) {
|
||
return $Object[$Name]
|
||
}
|
||
|
||
$property = $Object.PSObject.Properties[$Name]
|
||
if ($property) {
|
||
return $property.Value
|
||
}
|
||
|
||
return $Default
|
||
}
|
||
|
||
function Format-Duration {
|
||
param([Parameter(Mandatory)][timespan]$Duration)
|
||
|
||
if ($Duration.TotalHours -ge 1) {
|
||
return '{0}h {1}m {2}s' -f [int]$Duration.TotalHours, $Duration.Minutes, $Duration.Seconds
|
||
}
|
||
if ($Duration.TotalMinutes -ge 1) {
|
||
return '{0}m {1}s' -f [int]$Duration.TotalMinutes, $Duration.Seconds
|
||
}
|
||
return '{0}s' -f [int]$Duration.TotalSeconds
|
||
}
|
||
|
||
function Test-IsSuccessfulStatus {
|
||
param([Parameter(Mandatory)][string]$Status)
|
||
|
||
return $Status.ToLowerInvariant() -in @('success', 'succeeded', 'skipped', 'neutral')
|
||
}
|
||
|
||
function Get-LogSummary {
|
||
param(
|
||
[Parameter(Mandatory)][string]$LogText,
|
||
[switch]$IgnoreErrors
|
||
)
|
||
|
||
if ([string]::IsNullOrWhiteSpace($LogText)) {
|
||
return @{
|
||
HasErrors = $false
|
||
HasWarnings = $false
|
||
Summary = 'No log summary available.'
|
||
}
|
||
}
|
||
|
||
$matchedLines = New-Object System.Collections.Generic.List[string]
|
||
$hasErrors = $false
|
||
$hasWarnings = $false
|
||
|
||
foreach ($line in ($LogText -split "`r?`n")) {
|
||
$trimmed = $line.Trim()
|
||
if ([string]::IsNullOrWhiteSpace($trimmed)) {
|
||
continue
|
||
}
|
||
|
||
$isError = $false
|
||
if (-not $IgnoreErrors) {
|
||
foreach ($pattern in $Config.ErrorPatterns) {
|
||
if ($trimmed -match $pattern) {
|
||
$isError = $true
|
||
$hasErrors = $true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
$isWarning = $false
|
||
foreach ($pattern in $Config.WarningPatterns) {
|
||
if ($trimmed -match $pattern) {
|
||
$isWarning = $true
|
||
$hasWarnings = $true
|
||
break
|
||
}
|
||
}
|
||
|
||
if (($isError -or $isWarning) -and $matchedLines.Count -lt 8) {
|
||
$matchedLines.Add($trimmed)
|
||
}
|
||
}
|
||
|
||
$summary = if ($matchedLines.Count -gt 0) {
|
||
($matchedLines | Select-Object -First 8) -join "`n"
|
||
}
|
||
else {
|
||
'No matching warning or error lines found.'
|
||
}
|
||
|
||
return @{
|
||
HasErrors = $hasErrors
|
||
HasWarnings = $hasWarnings
|
||
Summary = $summary
|
||
}
|
||
}
|
||
|
||
function Get-Severity {
|
||
param(
|
||
[Parameter(Mandatory)][string]$Status,
|
||
[Parameter(Mandatory)][bool]$HasErrors,
|
||
[Parameter(Mandatory)][bool]$HasWarnings,
|
||
[Parameter(Mandatory)][bool]$IsSlow
|
||
)
|
||
|
||
$normalized = $Status.ToLowerInvariant()
|
||
if ($normalized -in @('failure', 'failed', 'cancelled', 'timed_out')) {
|
||
return 'Critical'
|
||
}
|
||
if ((-not (Test-IsSuccessfulStatus -Status $Status)) -and $HasErrors) {
|
||
return 'Critical'
|
||
}
|
||
if ($IsSlow -or $HasWarnings) {
|
||
return 'Warning'
|
||
}
|
||
return 'Info'
|
||
}
|
||
|
||
function Analyze-Build {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Job
|
||
)
|
||
|
||
$conclusion = Get-ObjectValue -Object $Job -Name 'conclusion'
|
||
$jobStatus = Get-ObjectValue -Object $Job -Name 'status'
|
||
$status = if ($conclusion) { [string]$conclusion } elseif ($jobStatus) { [string]$jobStatus } else { 'unknown' }
|
||
$isSuccessfulStatus = Test-IsSuccessfulStatus -Status $status
|
||
$duration = Get-Duration -Job $Job
|
||
$isSlow = $duration.TotalMinutes -ge [double]$Config.SlowBuildThresholdMins
|
||
$logText = ''
|
||
|
||
$definitelyProblematicStatus = $status.ToLowerInvariant() -in @('failure', 'failed', 'cancelled', 'timed_out')
|
||
if ($definitelyProblematicStatus -or $Config.NotifyWarningsAndSlow) {
|
||
$logText = Get-JobLogText -Repository $Repository -Job $Job
|
||
}
|
||
|
||
$logSummary = Get-LogSummary -LogText $logText -IgnoreErrors:$isSuccessfulStatus
|
||
$severity = Get-Severity -Status $status -HasErrors $logSummary.HasErrors -HasWarnings $logSummary.HasWarnings -IsSlow $isSlow
|
||
|
||
$shouldAlert = $severity -ne 'Info'
|
||
if (-not $Config.NotifyWarningsAndSlow -and $severity -eq 'Warning') {
|
||
$shouldAlert = $false
|
||
}
|
||
|
||
$sha = [string](Get-ObjectValue -Object $Job -Name 'head_sha' -Default '')
|
||
$commitMessage = Get-CommitMessage -Repository $Repository -Sha $sha
|
||
$runId = [string](Get-ObjectValue -Object $Job -Name 'run_id' -Default (Get-ObjectValue -Object $Job -Name 'id'))
|
||
$htmlUrl = [string](Get-ObjectValue -Object $Job -Name 'html_url' -Default '')
|
||
$runUrl = [string](Get-ObjectValue -Object $Job -Name 'run_url' -Default '')
|
||
$repositorySlug = Get-RepositorySlug -Repository $Repository
|
||
$buildUrl = if ($htmlUrl) { $htmlUrl } elseif ($runUrl) { $runUrl } else { Join-Url -Base $Config.GiteaBaseUrl -Path "$repositorySlug/actions/runs/$runId" }
|
||
$startedAt = Get-ObjectValue -Object $Job -Name 'started_at'
|
||
$createdAt = Get-ObjectValue -Object $Job -Name 'created_at'
|
||
|
||
return [pscustomobject]@{
|
||
Repository = $repositorySlug
|
||
Owner = Get-RepositoryOwner -Repository $Repository
|
||
Repo = Get-RepositoryName -Repository $Repository
|
||
JobId = [string](Get-ObjectValue -Object $Job -Name 'id')
|
||
RunId = $runId
|
||
Branch = [string](Get-ObjectValue -Object $Job -Name 'head_branch' -Default 'unknown')
|
||
CommitSha = $sha
|
||
CommitMessage = $commitMessage
|
||
Status = $status
|
||
Duration = $duration
|
||
DurationText = Format-Duration -Duration $duration
|
||
BuildUrl = $buildUrl
|
||
ErrorSummary = $logSummary.Summary
|
||
HasErrors = $logSummary.HasErrors
|
||
HasWarnings = $logSummary.HasWarnings
|
||
IsSlow = $isSlow
|
||
Severity = $severity
|
||
ShouldAlert = $shouldAlert
|
||
StartedAt = if ($startedAt) { [datetime]$startedAt } elseif ($createdAt) { [datetime]$createdAt } else { Get-Date }
|
||
}
|
||
}
|
||
|
||
function New-DiscordEmbed {
|
||
param([Parameter(Mandatory)][object]$Analysis)
|
||
|
||
$emoji = switch ($Analysis.Severity) {
|
||
'Critical' { '❌' }
|
||
'Warning' { '⚠️' }
|
||
default { 'ℹ️' }
|
||
}
|
||
|
||
$color = switch ($Analysis.Severity) {
|
||
'Critical' { 15158332 }
|
||
'Warning' { 16776960 }
|
||
default { 3447003 }
|
||
}
|
||
|
||
$shortSha = if ($Analysis.CommitSha.Length -ge 8) { $Analysis.CommitSha.Substring(0, 8) } else { $Analysis.CommitSha }
|
||
$commitValue = if ($shortSha) {
|
||
if ($Analysis.CommitMessage) { "``$shortSha`` $($Analysis.CommitMessage)" } else { "``$shortSha``" }
|
||
}
|
||
else {
|
||
'unknown'
|
||
}
|
||
|
||
$flags = @()
|
||
if ($Analysis.HasErrors) { $flags += 'errors found in logs' }
|
||
if ($Analysis.HasWarnings) { $flags += 'warnings found in logs' }
|
||
if ($Analysis.IsSlow) { $flags += "duration exceeded $($Config.SlowBuildThresholdMins)m" }
|
||
if ($flags.Count -eq 0) { $flags += 'problematic status' }
|
||
|
||
$summary = $Analysis.ErrorSummary
|
||
if ($summary.Length -gt 950) {
|
||
$summary = $summary.Substring(0, 947) + '...'
|
||
}
|
||
|
||
return @{
|
||
title = "$emoji $($Analysis.Severity): $($Analysis.Repository) build #$($Analysis.RunId)"
|
||
url = $Analysis.BuildUrl
|
||
color = $color
|
||
description = ($flags -join ', ')
|
||
fields = @(
|
||
@{ name = 'Status'; value = "$emoji $($Analysis.Status)"; inline = $true },
|
||
@{ name = 'Branch'; value = $Analysis.Branch; inline = $true },
|
||
@{ name = 'Duration'; value = $Analysis.DurationText; inline = $true },
|
||
@{ name = 'Commit'; value = $commitValue; inline = $false },
|
||
@{ name = 'Summary'; value = $summary; inline = $false }
|
||
)
|
||
footer = @{
|
||
text = "Job ID $($Analysis.JobId) | Run ID $($Analysis.RunId)"
|
||
}
|
||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||
}
|
||
}
|
||
|
||
function Send-DiscordNotification {
|
||
param([Parameter(Mandatory)][object[]]$Analyses)
|
||
|
||
$alerting = @($Analyses | Where-Object { $_.ShouldAlert } | Sort-Object Severity, Repository, RunId)
|
||
if ($alerting.Count -eq 0) {
|
||
Write-Log -Message 'No alertable builds found.'
|
||
return
|
||
}
|
||
|
||
$maxEmbeds = [int]$Config.MaxDiscordEmbeds
|
||
if ($maxEmbeds -lt 1) {
|
||
$maxEmbeds = 1
|
||
}
|
||
for ($offset = 0; $offset -lt $alerting.Count; $offset += $maxEmbeds) {
|
||
$chunk = @($alerting | Select-Object -Skip $offset -First $maxEmbeds)
|
||
$criticalCount = @($chunk | Where-Object { $_.Severity -eq 'Critical' }).Count
|
||
$warningCount = @($chunk | Where-Object { $_.Severity -eq 'Warning' }).Count
|
||
$embeds = @($chunk | ForEach-Object { New-DiscordEmbed -Analysis $_ })
|
||
$payload = @{
|
||
username = 'Gitea Build Monitor'
|
||
content = if ($criticalCount -gt 0) { "❌ Gitea build monitor found $criticalCount critical issue(s) and $warningCount warning(s)." } else { "⚠️ Gitea build monitor found $warningCount warning(s)." }
|
||
embeds = $embeds
|
||
}
|
||
|
||
Invoke-DiscordWebhook -Payload $payload
|
||
if ($DryRun) {
|
||
Write-Log -Message "Prepared Discord notification with $($chunk.Count) embed(s) in dry-run mode."
|
||
}
|
||
else {
|
||
Write-Log -Message "Sent Discord notification with $($chunk.Count) embed(s)."
|
||
}
|
||
}
|
||
}
|
||
|
||
function Get-PythonCommand {
|
||
$linuxVenvPython = Join-Path $PSScriptRoot '.venv/bin/python'
|
||
$windowsVenvPython = Join-Path $PSScriptRoot '.venv/Scripts/python.exe'
|
||
|
||
if (Test-Path -Path $linuxVenvPython) {
|
||
return $linuxVenvPython
|
||
}
|
||
if (Test-Path -Path $windowsVenvPython) {
|
||
return $windowsVenvPython
|
||
}
|
||
|
||
return 'python3'
|
||
}
|
||
|
||
function Invoke-AuthoringServerTests {
|
||
if (-not $Config.AuthoringTestsEnabled) {
|
||
return $null
|
||
}
|
||
|
||
if (-not (Test-Path -Path $Config.AuthoringTestsDirectory)) {
|
||
throw "Authoring test directory does not exist: $($Config.AuthoringTestsDirectory)"
|
||
}
|
||
|
||
$python = Get-PythonCommand
|
||
$arguments = @(
|
||
'-m',
|
||
'unittest',
|
||
'discover',
|
||
'-s',
|
||
$Config.AuthoringTestsDirectory,
|
||
'-p',
|
||
$Config.AuthoringTestsPattern,
|
||
'-v'
|
||
)
|
||
|
||
Write-Log -Message "Running authoring server tests with $python."
|
||
|
||
$process = New-Object System.Diagnostics.Process
|
||
$process.StartInfo.FileName = $python
|
||
foreach ($argument in $arguments) {
|
||
$process.StartInfo.ArgumentList.Add([string]$argument)
|
||
}
|
||
$process.StartInfo.WorkingDirectory = $PSScriptRoot
|
||
$process.StartInfo.UseShellExecute = $false
|
||
$process.StartInfo.RedirectStandardOutput = $true
|
||
$process.StartInfo.RedirectStandardError = $true
|
||
|
||
[void]$process.Start()
|
||
$stdoutTask = $process.StandardOutput.ReadToEndAsync()
|
||
$stderrTask = $process.StandardError.ReadToEndAsync()
|
||
$completed = $process.WaitForExit([int]$Config.AuthoringTestTimeoutSec * 1000)
|
||
if (-not $completed) {
|
||
$process.Kill()
|
||
$process.WaitForExit()
|
||
}
|
||
else {
|
||
$process.WaitForExit()
|
||
}
|
||
|
||
$stdout = $stdoutTask.Result
|
||
$stderr = $stderrTask.Result
|
||
$output = (($stdout, $stderr) -join "`n").Trim()
|
||
$exitCode = if ($completed) { $process.ExitCode } else { 124 }
|
||
$ran = 0
|
||
$failures = 0
|
||
$errors = 0
|
||
$skipped = 0
|
||
|
||
if ($output -match 'Ran\s+(\d+)\s+tests?') {
|
||
$ran = [int]$Matches[1]
|
||
}
|
||
if ($output -match 'failures=(\d+)') {
|
||
$failures = [int]$Matches[1]
|
||
}
|
||
if ($output -match 'errors=(\d+)') {
|
||
$errors = [int]$Matches[1]
|
||
}
|
||
if ($output -match 'skipped=(\d+)') {
|
||
$skipped = [int]$Matches[1]
|
||
}
|
||
|
||
$status = if ($completed -and $exitCode -eq 0) { 'passed' } elseif (-not $completed) { 'timed out' } else { 'failed' }
|
||
$tailLines = @($output -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -Last 12)
|
||
$summary = if ($tailLines.Count -gt 0) { $tailLines -join "`n" } else { 'No test output captured.' }
|
||
if ($summary.Length -gt 950) {
|
||
$summary = $summary.Substring(0, 947) + '...'
|
||
}
|
||
|
||
$result = [pscustomobject]@{
|
||
Name = 'authoring_server.py tests'
|
||
Status = $status
|
||
Success = ($completed -and $exitCode -eq 0)
|
||
ExitCode = $exitCode
|
||
Ran = $ran
|
||
Failures = $failures
|
||
Errors = $errors
|
||
Skipped = $skipped
|
||
Summary = $summary
|
||
Completed = $completed
|
||
Timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||
}
|
||
|
||
Write-Log -Message ("Authoring server tests {0}: ran={1}, failures={2}, errors={3}, skipped={4}, exit={5}" -f $result.Status, $result.Ran, $result.Failures, $result.Errors, $result.Skipped, $result.ExitCode)
|
||
return $result
|
||
}
|
||
|
||
function Send-AuthoringTestNotification {
|
||
param([Parameter(Mandatory)][object]$Result)
|
||
|
||
$emoji = if ($Result.Success) { '✅' } else { '❌' }
|
||
$color = if ($Result.Success) { 3066993 } else { 15158332 }
|
||
$payload = @{
|
||
username = 'Gitea Build Monitor'
|
||
content = "$emoji Authoring server test result: $($Result.Status)."
|
||
embeds = @(
|
||
@{
|
||
title = "$emoji $($Result.Name)"
|
||
color = $color
|
||
description = "Unit test results from gitea-build-monitor.ps1"
|
||
fields = @(
|
||
@{ name = 'Status'; value = $Result.Status; inline = $true },
|
||
@{ name = 'Tests'; value = [string]$Result.Ran; inline = $true },
|
||
@{ name = 'Exit code'; value = [string]$Result.ExitCode; inline = $true },
|
||
@{ name = 'Failures'; value = [string]$Result.Failures; inline = $true },
|
||
@{ name = 'Errors'; value = [string]$Result.Errors; inline = $true },
|
||
@{ name = 'Skipped'; value = [string]$Result.Skipped; inline = $true },
|
||
@{ name = 'Summary'; value = $Result.Summary; inline = $false }
|
||
)
|
||
timestamp = $Result.Timestamp
|
||
}
|
||
)
|
||
}
|
||
|
||
Invoke-DiscordWebhook -Payload $payload
|
||
if ($DryRun) {
|
||
Write-Log -Message 'Prepared authoring server test notification in dry-run mode.'
|
||
}
|
||
else {
|
||
Write-Log -Message 'Sent authoring server test notification.'
|
||
}
|
||
}
|
||
|
||
function Test-IsRecentBuild {
|
||
param([Parameter(Mandatory)][object]$Job)
|
||
|
||
$cutoff = (Get-Date).ToUniversalTime().AddMinutes(-[int]$Config.LookbackMinutes)
|
||
$candidateDates = @(
|
||
Get-ObjectValue -Object $Job -Name 'started_at'
|
||
Get-ObjectValue -Object $Job -Name 'completed_at'
|
||
Get-ObjectValue -Object $Job -Name 'created_at'
|
||
) | Where-Object { $_ }
|
||
foreach ($date in $candidateDates) {
|
||
if (([datetime]$date).ToUniversalTime() -ge $cutoff) {
|
||
return $true
|
||
}
|
||
}
|
||
return $false
|
||
}
|
||
|
||
function Get-CacheKey {
|
||
param(
|
||
[Parameter(Mandatory)][hashtable]$Repository,
|
||
[Parameter(Mandatory)][object]$Job
|
||
)
|
||
|
||
$jobId = Get-ObjectValue -Object $Job -Name 'id'
|
||
$conclusion = Get-ObjectValue -Object $Job -Name 'conclusion'
|
||
$status = Get-ObjectValue -Object $Job -Name 'status' -Default 'unknown'
|
||
$state = if ($conclusion) { $conclusion } else { $status }
|
||
|
||
return "$(Get-RepositorySlug -Repository $Repository):job:$jobId`:state:$state"
|
||
}
|
||
|
||
function Start-BuildMonitor {
|
||
Assert-Config
|
||
|
||
$logDirectory = Split-Path -Parent $Config.LogFile
|
||
if ($logDirectory -and -not (Test-Path -Path $logDirectory)) {
|
||
New-Item -ItemType Directory -Path $logDirectory -Force | Out-Null
|
||
}
|
||
|
||
if ($ValidateOnly) {
|
||
Write-Log -Message 'Validation mode completed: configuration and PowerShell runtime are valid.'
|
||
return
|
||
}
|
||
|
||
$authoringTestResult = Invoke-AuthoringServerTests
|
||
if ($null -ne $authoringTestResult) {
|
||
Send-AuthoringTestNotification -Result $authoringTestResult
|
||
}
|
||
|
||
$cache = if ($DryRun) { @{} } else { Read-ProcessedCache }
|
||
$analyses = New-Object System.Collections.Generic.List[object]
|
||
|
||
foreach ($repository in $Config.Repositories) {
|
||
Write-Log -Message "Fetching recent Gitea Actions jobs for $(Get-RepositorySlug -Repository $repository)."
|
||
|
||
try {
|
||
$jobs = Get-BuildRuns -Repository $repository
|
||
}
|
||
catch {
|
||
Write-Log -Level 'ERROR' -Message "Failed to retrieve jobs for $(Get-RepositorySlug -Repository $repository): $($_.Exception.Message)"
|
||
continue
|
||
}
|
||
|
||
foreach ($job in $jobs) {
|
||
if (-not (Test-IsRecentBuild -Job $job)) {
|
||
continue
|
||
}
|
||
|
||
$cacheKey = Get-CacheKey -Repository $repository -Job $job
|
||
if ($cache.ContainsKey($cacheKey)) {
|
||
continue
|
||
}
|
||
|
||
try {
|
||
$analysis = Analyze-Build -Repository $repository -Job $job
|
||
$analyses.Add($analysis)
|
||
if (-not $DryRun) {
|
||
$cache[$cacheKey] = Get-Date
|
||
}
|
||
Write-Log -Message ("Analyzed job {0} for {1}: severity={2}, status={3}, duration={4}" -f (Get-ObjectValue -Object $job -Name 'id'), $analysis.Repository, $analysis.Severity, $analysis.Status, $analysis.DurationText)
|
||
}
|
||
catch {
|
||
Write-Log -Level 'ERROR' -Message "Failed to analyze job $(Get-ObjectValue -Object $job -Name 'id'): $($_.Exception.Message)"
|
||
}
|
||
}
|
||
}
|
||
|
||
if ($analyses.Count -gt 0) {
|
||
Send-DiscordNotification -Analyses $analyses.ToArray()
|
||
}
|
||
else {
|
||
Write-Log -Message "No new jobs found in the last $($Config.LookbackMinutes) minute(s)."
|
||
}
|
||
|
||
if (-not $DryRun) {
|
||
Save-ProcessedCache -Cache $cache
|
||
}
|
||
|
||
if ($null -ne $authoringTestResult -and -not $authoringTestResult.Success) {
|
||
throw 'Authoring server tests failed.'
|
||
}
|
||
}
|
||
|
||
try {
|
||
Start-BuildMonitor
|
||
}
|
||
catch {
|
||
Write-Log -Level 'ERROR' -Message "Build monitor failed: $($_.Exception.Message)"
|
||
Write-Log -Level 'ERROR' -Message "Exception type: $($_.Exception.GetType().FullName)"
|
||
if ($_.ScriptStackTrace) {
|
||
Write-Log -Level 'ERROR' -Message "Stack trace: $($_.ScriptStackTrace)"
|
||
}
|
||
exit 1
|
||
}
|