@@ -1,11 +1,11 @@
<#
<#
. SYNOPSIS
. SYNOPSIS
Monitor recent Gitea Actions build job s and notify Discord about failures, warnings, errors, or slow build s .
Monitor recent Gitea Actions build run s and notify Discord about build statu s .
. DESCRIPTION
. DESCRIPTION
This script is intended to run after the main build pipeline . It queries a self-hosted Gitea instance,
This script is intended to run after the main build pipeline . It queries a self-hosted Gitea instance,
analyzes recent Actions job s for one or more repositories, parses logs where available, caches processed
analyzes recent Actions run s for one or more repositories, parses logs where available, caches processed
job IDs locally, and sends grouped Discord embed notifications .
run IDs locally, and sends grouped Discord embed notifications .
Required token permissions depend on your Gitea version and repository visibility . For private repos,
Required token permissions depend on your Gitea version and repository visibility . For private repos,
use a personal access token that can read repository Actions/jobs and logs .
use a personal access token that can read repository Actions/jobs and logs .
@@ -36,9 +36,11 @@ param(
Set-StrictMode -Version Latest
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$ErrorActionPreference = 'Stop'
$GiteaApiToken = if ( -not [ string ] :: IsNullOrWhiteSpace ( $env:API_KEY ) ) { $env:API_KEY } else { $env:GITEA_API_TOKEN }
$Config = @ {
$Config = @ {
GiteaBaseUrl = 'http://127.0.0.1:3000'
GiteaBaseUrl = 'http://127.0.0.1:3000'
ApiToken = $env:API_KEY
ApiToken = $GiteaApiToken
DiscordWebhookUrl = $env:DISCORD_WEBHOOK_URL
DiscordWebhookUrl = $env:DISCORD_WEBHOOK_URL
# Supports multiple repositories.
# Supports multiple repositories.
@@ -55,7 +57,7 @@ $Config = @{
MaxDiscordEmbeds = 10
MaxDiscordEmbeds = 10
RetryCount = 3
RetryCount = 3
RetryDelaySeconds = 2
RetryDelaySeconds = 2
AuthoringTestsEnabled = $tru e
AuthoringTestsEnabled = $fals e
AuthoringTestsPattern = 'test_authoring_server.py'
AuthoringTestsPattern = 'test_authoring_server.py'
AuthoringTestsDirectory = Join-Path $PSScriptRoot 'tests'
AuthoringTestsDirectory = Join-Path $PSScriptRoot 'tests'
AuthoringTestTimeoutSec = 120
AuthoringTestTimeoutSec = 120
@@ -71,6 +73,9 @@ $Config = @{
'(?im)\bdeprecated\b'
'(?im)\bdeprecated\b'
)
)
# When true, completed successful runs are reported too.
NotifySuccessfulBuilds = $true
# When true, successful but slow/warned jobs are reported too.
# When true, successful but slow/warned jobs are reported too.
# Keep this false if Discord should only receive notifications for failed/cancelled/timed-out jobs.
# Keep this false if Discord should only receive notifications for failed/cancelled/timed-out jobs.
NotifyWarningsAndSlow = $false
NotifyWarningsAndSlow = $false
@@ -244,11 +249,31 @@ function Get-BuildRuns {
$owner = [ uri ] :: EscapeDataString ( ( Get-RepositoryOwner -Repository $Repository ) )
$owner = [ uri ] :: EscapeDataString ( ( Get-RepositoryOwner -Repository $Repository ) )
$repo = [ uri ] :: EscapeDataString ( ( Get-RepositoryName -Repository $Repository ) )
$repo = [ uri ] :: EscapeDataString ( ( Get-RepositoryName -Repository $Repository ) )
$path = " /api/v1/repos/ $owner / $repo /actions/jobs?page=1&limit= $( $Config . MaxJobsPerRepository) "
$limit = [ int ] $Config . MaxJobsPerRepository
$response = Invoke-GiteaApi -Path $path
$jobs = Get-ObjectValue -Object $response -Name 'jobs'
try {
$runsResponse = Invoke-GiteaApi -Path " /api/v1/repos/ $owner / $repo /actions/runs?page=1&limit= $limit "
$runs = if ( $runsResponse -is [ array ] ) { $runsResponse } else { Get-ObjectValue -Object $runsResponse -Name 'workflow_runs' }
if ( -not $runs ) {
$runs = Get-ObjectValue -Object $runsResponse -Name 'runs'
}
if ( $runs ) {
foreach ( $run in @ ( $runs ) ) {
Add-Member -InputObject $run -NotePropertyName '_monitor_kind' -NotePropertyValue 'run' -Force
}
return @ ( $runs )
}
}
catch {
Write-Log -Level 'WARN' -Message " Could not fetch workflow runs for $( Get-RepositorySlug -Repository $Repository ) ; falling back to jobs endpoint: $( $_ . Exception . Message ) "
}
$jobsResponse = Invoke-GiteaApi -Path " /api/v1/repos/ $owner / $repo /actions/jobs?page=1&limit= $limit "
$jobs = if ( $jobsResponse -is [ array ] ) { $jobsResponse } else { Get-ObjectValue -Object $jobsResponse -Name 'jobs' }
if ( $jobs ) {
if ( $jobs ) {
foreach ( $job in @ ( $jobs ) ) {
Add-Member -InputObject $job -NotePropertyName '_monitor_kind' -NotePropertyValue 'job' -Force
}
return @ ( $jobs )
return @ ( $jobs )
}
}
@@ -311,6 +336,55 @@ function Get-JobLogText {
}
}
}
}
function Get-BuildLogText {
param (
[ Parameter ( Mandatory ) ] [ hashtable ] $Repository ,
[ Parameter ( Mandatory ) ] [ object ] $Build
)
$kind = [ string ] ( Get-ObjectValue -Object $Build -Name '_monitor_kind' -Default 'job' )
if ( $kind -ne 'run' ) {
return Get-JobLogText -Repository $Repository -Job $Build
}
try {
$owner = [ uri ] :: EscapeDataString ( ( Get-RepositoryOwner -Repository $Repository ) )
$repo = [ uri ] :: EscapeDataString ( ( Get-RepositoryName -Repository $Repository ) )
$runId = [ uri ] :: EscapeDataString ( [ string ] ( Get-ObjectValue -Object $Build -Name 'id' ) )
$response = Invoke-GiteaApi -Path " /api/v1/repos/ $owner / $repo /actions/runs/ $runId /jobs "
$jobs = Get-ObjectValue -Object $response -Name 'jobs'
if ( -not $jobs ) {
return ''
}
$logParts = New-Object System . Collections . Generic . List [ string ]
foreach ( $job in @ ( $jobs ) ) {
$conclusion = Get-ObjectValue -Object $job -Name 'conclusion'
$jobStatus = Get-ObjectValue -Object $job -Name 'status'
$status = if ( $conclusion ) { [ string ] $conclusion } elseif ( $jobStatus ) { [ string ] $jobStatus } else { 'unknown' }
if ( $status . ToLowerInvariant ( ) -notin @ ( 'failure' , 'failed' , 'cancelled' , 'canceled' , 'timed_out' ) ) {
continue
}
$jobName = [ string ] ( Get-ObjectValue -Object $job -Name 'name' -Default ( Get-ObjectValue -Object $job -Name 'id' ) )
$jobLog = Get-JobLogText -Repository $Repository -Job $job
if ( -not [ string ] :: IsNullOrWhiteSpace ( $jobLog ) ) {
$logParts . Add ( ( " Job {0}: `n {1} " -f $jobName , $jobLog ) )
}
}
$text = ( $logParts -join " `n `n " )
if ( $text . Length -gt [ int ] $Config . MaxLogBytesToAnalyze ) {
return $text . Substring ( [ Math ] :: Max ( 0 , $text . Length - [ int ] $Config . MaxLogBytesToAnalyze ) )
}
return $text
}
catch {
Write-Log -Level 'WARN' -Message " Could not fetch logs for run $( Get-ObjectValue -Object $Build -Name 'id' ) : $( $_ . Exception . Message ) "
return ''
}
}
function Get-Duration {
function Get-Duration {
param ( [ Parameter ( Mandatory ) ] [ object ] $Job )
param ( [ Parameter ( Mandatory ) ] [ object ] $Job )
@@ -383,6 +457,12 @@ function Test-IsSuccessfulStatus {
return $Status . ToLowerInvariant ( ) -in @ ( 'success' , 'succeeded' , 'skipped' , 'neutral' )
return $Status . ToLowerInvariant ( ) -in @ ( 'success' , 'succeeded' , 'skipped' , 'neutral' )
}
}
function Test-IsCompletedStatus {
param ( [ Parameter ( Mandatory ) ] [ string ] $Status )
return $Status . ToLowerInvariant ( ) -in @ ( 'success' , 'succeeded' , 'skipped' , 'neutral' , 'failure' , 'failed' , 'cancelled' , 'canceled' , 'timed_out' , 'completed' )
}
function Get-LogSummary {
function Get-LogSummary {
param (
param (
[ Parameter ( Mandatory ) ] [ string ] $LogText ,
[ Parameter ( Mandatory ) ] [ string ] $LogText ,
@@ -455,7 +535,7 @@ function Get-Severity {
)
)
$normalized = $Status . ToLowerInvariant ( )
$normalized = $Status . ToLowerInvariant ( )
if ( $normalized -in @ ( 'failure' , 'failed' , 'cancelled' , 'timed_out' ) ) {
if ( $normalized -in @ ( 'failure' , 'failed' , 'cancelled' , 'canceled' , 'timed_out' ) ) {
return 'Critical'
return 'Critical'
}
}
if ( ( -not ( Test-IsSuccessfulStatus -Status $Status ) ) -and $HasErrors ) {
if ( ( -not ( Test-IsSuccessfulStatus -Status $Status ) ) -and $HasErrors ) {
@@ -481,9 +561,9 @@ function Analyze-Build {
$isSlow = $duration . TotalMinutes -ge [ double ] $Config . SlowBuildThresholdMins
$isSlow = $duration . TotalMinutes -ge [ double ] $Config . SlowBuildThresholdMins
$logText = ''
$logText = ''
$definitelyProblematicStatus = $status . ToLowerInvariant ( ) -in @ ( 'failure' , 'failed' , 'cancelled' , 'timed_out' )
$definitelyProblematicStatus = $status . ToLowerInvariant ( ) -in @ ( 'failure' , 'failed' , 'cancelled' , 'canceled' , 'timed_out' )
if ( $definitelyProblematicStatus -or $Config . NotifyWarningsAndSlow ) {
if ( $definitelyProblematicStatus -or $Config . NotifyWarningsAndSlow ) {
$logText = Get-Job LogText -Repository $Repository -Job $Job
$logText = Get-Build LogText -Repository $Repository -Build $Job
}
}
$logSummary = Get-LogSummary -LogText $logText -IgnoreErrors: $isSuccessfulStatus
$logSummary = Get-LogSummary -LogText $logText -IgnoreErrors: $isSuccessfulStatus
@@ -493,6 +573,9 @@ function Analyze-Build {
if ( -not $Config . NotifyWarningsAndSlow -and $severity -eq 'Warning' ) {
if ( -not $Config . NotifyWarningsAndSlow -and $severity -eq 'Warning' ) {
$shouldAlert = $false
$shouldAlert = $false
}
}
if ( -not $shouldAlert -and $Config . NotifySuccessfulBuilds -and ( Test-IsCompletedStatus -Status $status ) ) {
$shouldAlert = $true
}
$sha = [ string ] ( Get-ObjectValue -Object $Job -Name 'head_sha' -Default '' )
$sha = [ string ] ( Get-ObjectValue -Object $Job -Name 'head_sha' -Default '' )
$commitMessage = Get-CommitMessage -Repository $Repository -Sha $sha
$commitMessage = Get-CommitMessage -Repository $Repository -Sha $sha
@@ -530,16 +613,27 @@ function Analyze-Build {
function New-DiscordEmbed {
function New-DiscordEmbed {
param ( [ Parameter ( Mandatory ) ] [ object ] $Analysis )
param ( [ Parameter ( Mandatory ) ] [ object ] $Analysis )
$emoji = switch ( $Analysis . Severity ) {
$isSuccess = Test-IsSuccessfulStatus -Status ( [ string ] $Analysis . Status )
'Critical' { '❌' }
$emoji = if ( $isSuccess ) {
'Warning' { '⚠️' }
'✅'
default { 'ℹ ️ ' }
}
else {
switch ( $Analysis . Severity ) {
'Critical' { '❌' }
'Warning' { '⚠️' }
default { 'ℹ ️ ' }
}
}
}
$color = switch ( $Analysis . Severity ) {
$color = if ( $isSuccess ) {
'Critical' { 15158332 }
3066993
'Warning' { 16776960 }
}
default { 3447003 }
else {
switch ( $Analysis . Severity ) {
'Critical' { 15158332 }
'Warning' { 16776960 }
default { 3447003 }
}
}
}
$shortSha = if ( $Analysis . CommitSha . Length -ge 8 ) { $Analysis . CommitSha . Substring ( 0 , 8 ) } else { $Analysis . CommitSha }
$shortSha = if ( $Analysis . CommitSha . Length -ge 8 ) { $Analysis . CommitSha . Substring ( 0 , 8 ) } else { $Analysis . CommitSha }
@@ -554,7 +648,14 @@ function New-DiscordEmbed {
if ( $Analysis . HasErrors ) { $flags + = 'errors found in logs' }
if ( $Analysis . HasErrors ) { $flags + = 'errors found in logs' }
if ( $Analysis . HasWarnings ) { $flags + = 'warnings found in logs' }
if ( $Analysis . HasWarnings ) { $flags + = 'warnings found in logs' }
if ( $Analysis . IsSlow ) { $flags + = " duration exceeded $( $Config . SlowBuildThresholdMins ) m " }
if ( $Analysis . IsSlow ) { $flags + = " duration exceeded $( $Config . SlowBuildThresholdMins ) m " }
if ( $flags . Count -eq 0 ) { $flags + = 'problematic status' }
if ( $flags . Count -eq 0 ) {
if ( $isSuccess ) {
$flags + = 'build completed successfully'
}
else {
$flags + = 'build status changed'
}
}
$summary = $Analysis . ErrorSummary
$summary = $Analysis . ErrorSummary
if ( $summary . Length -gt 950 ) {
if ( $summary . Length -gt 950 ) {
@@ -562,7 +663,7 @@ function New-DiscordEmbed {
}
}
return @ {
return @ {
title = " $emoji $( $Analysis . Severity ) : $( $Analysis . Repository ) build # $( $Analysis . RunId ) "
title = " $emoji $( $Analysis . Status ) : $( $Analysis . Repository ) build # $( $Analysis . RunId ) "
url = $Analysis . BuildUrl
url = $Analysis . BuildUrl
color = $color
color = $color
description = ( $flags -join ', ' )
description = ( $flags -join ', ' )
@@ -597,10 +698,23 @@ function Send-DiscordNotification {
$chunk = @ ( $alerting | Select-Object -Skip $offset -First $maxEmbeds )
$chunk = @ ( $alerting | Select-Object -Skip $offset -First $maxEmbeds )
$criticalCount = @ ( $chunk | Where-Object { $_ . Severity -eq 'Critical' } ) . Count
$criticalCount = @ ( $chunk | Where-Object { $_ . Severity -eq 'Critical' } ) . Count
$warningCount = @ ( $chunk | Where-Object { $_ . Severity -eq 'Warning' } ) . Count
$warningCount = @ ( $chunk | Where-Object { $_ . Severity -eq 'Warning' } ) . Count
$successCount = @ ( $chunk | Where-Object { Test-IsSuccessfulStatus -Status ( [ string ] $_ . Status ) } ) . Count
$completedCount = $chunk . Count
$embeds = @ ( $chunk | ForEach-Object { New-DiscordEmbed -Analysis $_ } )
$embeds = @ ( $chunk | ForEach-Object { New-DiscordEmbed -Analysis $_ } )
$payload = @ {
$payload = @ {
username = 'Gitea Build Monitor'
username = 'Gitea Build Monitor'
content = if ( $criticalCount -gt 0 ) { " ❌ Gitea build monitor found $criticalCount critical issue(s) and $warningCount warning(s). " } else { " ⚠️ Gitea build monitor found $warningCount warning(s). " }
content = if ( $criticalCount -gt 0 ) {
" ❌ Gitea build monitor found $criticalCount failed build(s), $warningCount warning(s), and $successCount successful build(s). "
}
elseif ( $warningCount -gt 0 ) {
" ⚠️ Gitea build monitor found $warningCount warning(s) and $successCount successful build(s). "
}
elseif ( $successCount -gt 0 ) {
" ✅ Gitea build monitor found $successCount successful build(s). "
}
else {
" ℹ ️ Gitea build monitor found $completedCount completed build(s). "
}
embeds = $embeds
embeds = $embeds
}
}
@@ -776,18 +890,35 @@ function Test-IsRecentBuild {
return $false
return $false
}
}
function Test-IsCompletedBuild {
param ( [ Parameter ( Mandatory ) ] [ object ] $Build )
$conclusion = Get-ObjectValue -Object $Build -Name 'conclusion'
if ( $conclusion ) {
return Test-IsCompletedStatus -Status ( [ string ] $conclusion )
}
$status = Get-ObjectValue -Object $Build -Name 'status'
if ( $status ) {
return Test-IsCompletedStatus -Status ( [ string ] $status )
}
return $false
}
function Get-CacheKey {
function Get-CacheKey {
param (
param (
[ Parameter ( Mandatory ) ] [ hashtable ] $Repository ,
[ Parameter ( Mandatory ) ] [ hashtable ] $Repository ,
[ Parameter ( Mandatory ) ] [ object ] $Job
[ Parameter ( Mandatory ) ] [ object ] $Job
)
)
$kind = Get-ObjectValue -Object $Job -Name '_monitor_kind' -Default 'job'
$jobId = Get-ObjectValue -Object $Job -Name 'id'
$jobId = Get-ObjectValue -Object $Job -Name 'id'
$conclusion = Get-ObjectValue -Object $Job -Name 'conclusion'
$conclusion = Get-ObjectValue -Object $Job -Name 'conclusion'
$status = Get-ObjectValue -Object $Job -Name 'status' -Default 'unknown'
$status = Get-ObjectValue -Object $Job -Name 'status' -Default 'unknown'
$state = if ( $conclusion ) { $conclusion } else { $status }
$state = if ( $conclusion ) { $conclusion } else { $status }
return " $( Get-RepositorySlug -Repository $Repository ) :job : $jobId ` :state: $state "
return " $( Get-RepositorySlug -Repository $Repository ) :$kind ` : $jobId ` :state: $state "
}
}
function Start-BuildMonitor {
function Start-BuildMonitor {
@@ -812,13 +943,13 @@ function Start-BuildMonitor {
$analyses = New-Object System . Collections . Generic . List [ object ]
$analyses = New-Object System . Collections . Generic . List [ object ]
foreach ( $repository in $Config . Repositories ) {
foreach ( $repository in $Config . Repositories ) {
Write-Log -Message " Fetching recent Gitea Actions job s for $( Get-RepositorySlug -Repository $repository ) . "
Write-Log -Message " Fetching recent Gitea Actions run s for $( Get-RepositorySlug -Repository $repository ) . "
try {
try {
$jobs = Get-BuildRuns -Repository $repository
$jobs = Get-BuildRuns -Repository $repository
}
}
catch {
catch {
Write-Log -Level 'ERROR' -Message " Failed to retrieve job s for $( Get-RepositorySlug -Repository $repository ) : $( $_ . Exception . Message ) "
Write-Log -Level 'ERROR' -Message " Failed to retrieve build s for $( Get-RepositorySlug -Repository $repository ) : $( $_ . Exception . Message ) "
continue
continue
}
}
@@ -826,6 +957,9 @@ function Start-BuildMonitor {
if ( -not ( Test-IsRecentBuild -Job $job ) ) {
if ( -not ( Test-IsRecentBuild -Job $job ) ) {
continue
continue
}
}
if ( -not ( Test-IsCompletedBuild -Build $job ) ) {
continue
}
$cacheKey = Get-CacheKey -Repository $repository -Job $job
$cacheKey = Get-CacheKey -Repository $repository -Job $job
if ( $cache . ContainsKey ( $cacheKey ) ) {
if ( $cache . ContainsKey ( $cacheKey ) ) {