Skip to content

Commit 7de1582

Browse files
arcuri82arcuri82
authored
Merge pull request #1513 from WebFuzzing/pr/counting-warnings-and-errors
Introduce Build metrics step and upgrade CI Actions to Node 24 compatible versions.
2 parents 565de8a + ecfcab5 commit 7de1582

2 files changed

Lines changed: 176 additions & 27 deletions

File tree

‎.github/actions/full-build-metrics/action.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: Full Build Metrics
2+
description: Count Maven warnings/errors, compare against latest successful baseline, and persist current metrics artifact
3+
4+
inputs:
5+
log-file:
6+
description: Path to the captured build log
7+
required: false
8+
default: full-build-base.log
9+
artifact-name:
10+
description: Artifact name used to store metrics
11+
required: false
12+
default: full-build-base-metrics
13+
baseline-branch:
14+
description: Branch to fetch the latest successful baseline metrics from
15+
required: false
16+
default: master
17+
workflow-file:
18+
description: Workflow filename to query for baseline artifacts
19+
required: false
20+
default: ci.yml
21+
retention-days:
22+
description: Artifact retention period in days
23+
required: false
24+
default: "90"
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Download previous metrics artifact
30+
uses: dawidd6/action-download-artifact@v20
31+
with:
32+
workflow: ${{ inputs.workflow-file }}
33+
workflow_conclusion: success
34+
branch: ${{ inputs.baseline-branch }}
35+
name: ${{ inputs.artifact-name }}
36+
path: previous-metrics
37+
if_no_artifact_found: warn
38+
39+
- name: Count warnings/errors and compare
40+
shell: bash
41+
run: |
42+
if [ -f "${{ inputs.log-file }}" ]; then
43+
warnings_count=$(grep -c "^\[WARNING\]" "${{ inputs.log-file }}" || true)
44+
errors_count=$(grep -c "^\[ERROR\]" "${{ inputs.log-file }}" || true)
45+
else
46+
warnings_count=9999999
47+
errors_count=9999999
48+
fi
49+
50+
previous_warnings=""
51+
previous_errors=""
52+
if [ -f previous-metrics/build-metrics.env ]; then
53+
# shellcheck disable=SC1091
54+
. previous-metrics/build-metrics.env
55+
previous_warnings="${WARNINGS:-}"
56+
previous_errors="${ERRORS:-}"
57+
fi
58+
59+
echo "full-build-base warnings: ${warnings_count}"
60+
echo "full-build-base errors: ${errors_count}"
61+
62+
regression=false
63+
if [ -n "${previous_warnings}" ] && [ -n "${previous_errors}" ]; then
64+
warnings_delta=$((warnings_count - previous_warnings))
65+
errors_delta=$((errors_count - previous_errors))
66+
echo "delta warnings vs last successful ${{ inputs.baseline-branch }} run: ${warnings_delta}"
67+
echo "delta errors vs last successful ${{ inputs.baseline-branch }} run: ${errors_delta}"
68+
if [ "${warnings_count}" -le "${previous_warnings}" ] && [ "${errors_count}" -le "${previous_errors}" ]; then
69+
echo "Result: green (no increase in warnings or errors)."
70+
else
71+
echo "Result: yellow (warnings or errors increased)."
72+
regression=true
73+
fi
74+
else
75+
echo "No previous metrics artifact found on ${{ inputs.baseline-branch }} for comparison."
76+
echo "Result: green (no baseline available)."
77+
fi
78+
79+
printf "WARNINGS=%s\nERRORS=%s\n" "${warnings_count}" "${errors_count}" > build-metrics.env
80+
81+
{
82+
echo "full-build-base warnings: ${warnings_count}"
83+
echo "full-build-base errors: ${errors_count}"
84+
if [ -n "${previous_warnings}" ] && [ -n "${previous_errors}" ]; then
85+
echo "delta warnings vs last successful ${{ inputs.baseline-branch }} run: ${warnings_delta}"
86+
echo "delta errors vs last successful ${{ inputs.baseline-branch }} run: ${errors_delta}"
87+
else
88+
echo "No previous metrics artifact found on ${{ inputs.baseline-branch }} for comparison."
89+
fi
90+
} >> "$GITHUB_STEP_SUMMARY"
91+
92+
if [ "${regression}" = "true" ]; then
93+
echo "METRICS_REGRESSION=true" >> "$GITHUB_ENV"
94+
else
95+
echo "METRICS_REGRESSION=false" >> "$GITHUB_ENV"
96+
fi
97+
98+
- name: Upload current metrics artifact
99+
uses: actions/upload-artifact@v7
100+
with:
101+
name: ${{ inputs.artifact-name }}
102+
path: build-metrics.env
103+
retention-days: ${{ inputs.retention-days }}
104+
if-no-files-found: error
105+
106+
- name: Mark metrics regression
107+
if: ${{ env.METRICS_REGRESSION == 'true' }}
108+
shell: bash
109+
run: exit 1

‎.github/workflows/ci.yml‎

Lines changed: 67 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ env:
4040
# https://github.com/WebFuzzing/EvoMaster/issues/447
4141
release-jdk: 21
4242
build-jdk: 17
43+
java-distribution: temurin
4344
retention-days: 5
4445
debug: false # put to true if need to debug a specific test
4546
debugTestName: "GeneRandomizedTest" # replace with test to debug
@@ -74,14 +75,15 @@ jobs:
7475
needs: setup
7576
if: needs.setup.outputs.debug == 'true'
7677
steps:
77-
- uses: actions/checkout@v4
78+
- uses: actions/checkout@v6
7879
- name: Setup JDK ${{env.build-jdk}}
7980
uses: actions/setup-java@v5
8081
with:
82+
distribution: ${{env.java-distribution}}
8183
java-version: ${{env.build-jdk}}
8284
distribution: 'temurin'
8385
- name: Cache Maven packages
84-
uses: actions/cache@v3
86+
uses: actions/cache@v5
8587
with:
8688
path: ~/.m2
8789
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -97,25 +99,37 @@ jobs:
9799
if: needs.setup.outputs.debug == 'false'
98100
steps:
99101
# Checkout code
100-
- uses: actions/checkout@v4
102+
- uses: actions/checkout@v6
101103
# Build/test for JDK
102104
- name: Setup JDK ${{env.build-jdk}}
103105
uses: actions/setup-java@v5
104106
with:
107+
distribution: ${{env.java-distribution}}
105108
java-version: ${{env.build-jdk}}
106109
distribution: 'temurin'
107110
- name: Cache Maven packages
108-
uses: actions/cache@v3
111+
uses: actions/cache@v5
109112
with:
110113
path: ~/.m2
111114
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
112115
restore-keys: ${{ runner.os }}-m2
113116
- name: Build with Maven
114-
run: mvn clean verify --fae
117+
# the set command makes the build fail if mvn command fails (no change in behavior)
118+
run: |
119+
set -o pipefail
120+
mvn clean verify --fae 2>&1 | tee full-build-base.log
115121
env:
116122
CI_env: GithubAction
123+
- name: Upload full-build-base log for metrics job
124+
if: success()
125+
uses: actions/upload-artifact@v7
126+
with:
127+
name: full-build-base-log-current
128+
path: full-build-base.log
129+
retention-days: 1
130+
if-no-files-found: error
117131
- name: Upload evomaster.jar
118-
uses: actions/upload-artifact@v4
132+
uses: actions/upload-artifact@v7
119133
with:
120134
name: evomaster.jar
121135
path: core/target/evomaster.jar
@@ -140,21 +154,42 @@ jobs:
140154
name: Uploading coverage to CodeCov is done only on 'master' branch builds
141155
run: echo Skipping upload to CodeCov
142156

143-
157+
full-build-metrics:
158+
runs-on: ubuntu-latest
159+
needs: full-build-base
160+
if: needs.full-build-base.result == 'success'
161+
steps:
162+
- uses: actions/checkout@v6
163+
- name: Download full-build-base log
164+
uses: actions/download-artifact@v8
165+
with:
166+
name: full-build-base-log-current
167+
path: .
168+
- name: Analyze and persist full-build-base metrics
169+
continue-on-error: true
170+
uses: ./.github/actions/full-build-metrics
171+
with:
172+
log-file: full-build-base.log
173+
artifact-name: full-build-base-metrics
174+
baseline-branch: master
175+
workflow-file: ci.yml
176+
#These metrics occupy very little space, increasing the retention so they are not lost if nothing is merged to master in a while
177+
retention-days: 90
144178

145179
base-build-mac:
146180
runs-on: macos-latest
147181
needs: setup
148182
if: needs.setup.outputs.debug == 'false'
149183
steps:
150-
- uses: actions/checkout@v4
184+
- uses: actions/checkout@v6
151185
- name: Setup JDK ${{env.release-jdk}}
152186
uses: actions/setup-java@v5
153187
with:
188+
distribution: ${{env.java-distribution}}
154189
java-version: ${{env.release-jdk}}
155190
distribution: 'temurin'
156191
- name: Cache Maven packages
157-
uses: actions/cache@v3
192+
uses: actions/cache@v5
158193
with:
159194
path: ~/.m2
160195
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -169,14 +204,15 @@ jobs:
169204
needs: setup
170205
if: needs.setup.outputs.debug == 'false'
171206
steps:
172-
- uses: actions/checkout@v4
207+
- uses: actions/checkout@v6
173208
- name: Setup JDK ${{env.release-jdk}}
174209
uses: actions/setup-java@v5
175210
with:
211+
distribution: ${{env.java-distribution}}
176212
java-version: ${{env.release-jdk}}
177213
distribution: 'temurin'
178214
- name: Cache Maven packages
179-
uses: actions/cache@v3
215+
uses: actions/cache@v5
180216
with:
181217
path: ~/.m2
182218
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -225,22 +261,23 @@ jobs:
225261
needs: full-build-base
226262
runs-on: windows-latest
227263
steps:
228-
- uses: actions/checkout@v4
264+
- uses: actions/checkout@v6
229265
- name: Setup JDK ${{env.release-jdk}}
230266
uses: actions/setup-java@v5
231267
with:
268+
distribution: ${{env.java-distribution}}
232269
java-version: ${{env.release-jdk}}
233270
distribution: 'temurin'
234271
- name: Download fat jar
235-
uses: actions/download-artifact@v4
272+
uses: actions/download-artifact@v8
236273
with:
237274
name: evomaster.jar
238275
path: core/target
239276
- name: Build installation file
240277
shell: bash
241278
run: bash makeExecutable.sh WINDOWS
242279
- name: Upload installation file
243-
uses: actions/upload-artifact@v4
280+
uses: actions/upload-artifact@v7
244281
with:
245282
name: evomaster.msi
246283
path: release/evomaster-${{env.evomaster-version}}.msi
@@ -251,22 +288,23 @@ jobs:
251288
needs: full-build-base
252289
runs-on: macos-latest
253290
steps:
254-
- uses: actions/checkout@v3
291+
- uses: actions/checkout@v6
255292
- name: Setup JDK ${{env.release-jdk}}
256293
uses: actions/setup-java@v5
257294
with:
295+
distribution: ${{env.java-distribution}}
258296
java-version: ${{env.release-jdk}}
259297
distribution: 'temurin'
260298
- name: Download fat jar
261-
uses: actions/download-artifact@v4
299+
uses: actions/download-artifact@v8
262300
with:
263301
name: evomaster.jar
264302
path: core/target
265303
- name: Build installation file
266304
shell: bash
267305
run: bash makeExecutable.sh OSX
268306
- name: Upload installation file
269-
uses: actions/upload-artifact@v4
307+
uses: actions/upload-artifact@v7
270308
with:
271309
name: evomaster.dmg
272310
path: release/evomaster-${{env.evomaster-version}}.dmg
@@ -277,14 +315,15 @@ jobs:
277315
needs: full-build-base
278316
runs-on: ubuntu-latest
279317
steps:
280-
- uses: actions/checkout@v4
318+
- uses: actions/checkout@v6
281319
- name: Setup JDK ${{env.release-jdk}}
282320
uses: actions/setup-java@v5
283321
with:
322+
distribution: ${{env.java-distribution}}
284323
java-version: ${{env.release-jdk}}
285324
distribution: 'temurin'
286325
- name: Download fat jar
287-
uses: actions/download-artifact@v4
326+
uses: actions/download-artifact@v8
288327
with:
289328
name: evomaster.jar
290329
path: core/target
@@ -295,7 +334,7 @@ jobs:
295334
shell: bash
296335
run: ls -l release
297336
- name: Upload installation file
298-
uses: actions/upload-artifact@v4
337+
uses: actions/upload-artifact@v7
299338
with:
300339
name: evomaster.deb
301340
# JDK 17 and 21 use different suffixes... doesn't seem configurable :(
@@ -308,11 +347,11 @@ jobs:
308347
test-utils-js:
309348
runs-on: ubuntu-latest
310349
steps:
311-
- uses: actions/checkout@v4
350+
- uses: actions/checkout@v6
312351
- name: Use Node.js
313-
uses: actions/setup-node@v4
352+
uses: actions/setup-node@v6
314353
with:
315-
node-version: 20
354+
node-version: 24
316355
- run: npm ci
317356
working-directory: ./test-utils/test-utils-js
318357
- run: npm test
@@ -322,9 +361,9 @@ jobs:
322361
test-utils-py:
323362
runs-on: ubuntu-latest
324363
steps:
325-
- uses: actions/checkout@v4
364+
- uses: actions/checkout@v6
326365
- name: Set up Python
327-
uses: actions/setup-python@v5
366+
uses: actions/setup-python@v6
328367
with:
329368
python-version: '3.10'
330369
- name: Install dependencies
@@ -345,14 +384,15 @@ jobs:
345384
needs: setup
346385
if: needs.setup.outputs.debug == 'false'
347386
steps:
348-
- uses: actions/checkout@v4
387+
- uses: actions/checkout@v6
349388
- name: Setup JDK ${{env.build-jdk}}
350389
uses: actions/setup-java@v5
351390
with:
391+
distribution: ${{env.java-distribution}}
352392
java-version: ${{env.build-jdk}}
353393
distribution: 'temurin'
354394
- name: Cache Maven packages
355-
uses: actions/cache@v3
395+
uses: actions/cache@v5
356396
with:
357397
path: ~/.m2
358398
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

0 commit comments

Comments
 (0)