Skip to content

Commit a8d6764

Browse files
arcuri82arcuri82
committed
Merge branch 'master' of https://github.com/WebFuzzing/EvoMaster into enable-new-security
2 parents 87c6577 + 9daba4f commit a8d6764

2 files changed

Lines changed: 176 additions & 35 deletions

File tree

‎.github/actions/full-build-metrics/action.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: Full Build Metrics
2+
description: Count Maven warnings/errors, compare against latest successful baseline, and persist current metrics artifact
3+
4+
inputs:
5+
log-file:
6+
description: Path to the captured build log
7+
required: false
8+
default: full-build-base.log
9+
artifact-name:
10+
description: Artifact name used to store metrics
11+
required: false
12+
default: full-build-base-metrics
13+
baseline-branch:
14+
description: Branch to fetch the latest successful baseline metrics from
15+
required: false
16+
default: master
17+
workflow-file:
18+
description: Workflow filename to query for baseline artifacts
19+
required: false
20+
default: ci.yml
21+
retention-days:
22+
description: Artifact retention period in days
23+
required: false
24+
default: "90"
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Download previous metrics artifact
30+
uses: dawidd6/action-download-artifact@v20
31+
with:
32+
workflow: ${{ inputs.workflow-file }}
33+
workflow_conclusion: success
34+
branch: ${{ inputs.baseline-branch }}
35+
name: ${{ inputs.artifact-name }}
36+
path: previous-metrics
37+
if_no_artifact_found: warn
38+
39+
- name: Count warnings/errors and compare
40+
shell: bash
41+
run: |
42+
if [ -f "${{ inputs.log-file }}" ]; then
43+
warnings_count=$(grep -c "^\[WARNING\]" "${{ inputs.log-file }}" || true)
44+
errors_count=$(grep -c "^\[ERROR\]" "${{ inputs.log-file }}" || true)
45+
else
46+
warnings_count=9999999
47+
errors_count=9999999
48+
fi
49+
50+
previous_warnings=""
51+
previous_errors=""
52+
if [ -f previous-metrics/build-metrics.env ]; then
53+
# shellcheck disable=SC1091
54+
. previous-metrics/build-metrics.env
55+
previous_warnings="${WARNINGS:-}"
56+
previous_errors="${ERRORS:-}"
57+
fi
58+
59+
echo "full-build-base warnings: ${warnings_count}"
60+
echo "full-build-base errors: ${errors_count}"
61+
62+
regression=false
63+
if [ -n "${previous_warnings}" ] && [ -n "${previous_errors}" ]; then
64+
warnings_delta=$((warnings_count - previous_warnings))
65+
errors_delta=$((errors_count - previous_errors))
66+
echo "delta warnings vs last successful ${{ inputs.baseline-branch }} run: ${warnings_delta}"
67+
echo "delta errors vs last successful ${{ inputs.baseline-branch }} run: ${errors_delta}"
68+
if [ "${warnings_count}" -le "${previous_warnings}" ] && [ "${errors_count}" -le "${previous_errors}" ]; then
69+
echo "Result: green (no increase in warnings or errors)."
70+
else
71+
echo "Result: yellow (warnings or errors increased)."
72+
regression=true
73+
fi
74+
else
75+
echo "No previous metrics artifact found on ${{ inputs.baseline-branch }} for comparison."
76+
echo "Result: green (no baseline available)."
77+
fi
78+
79+
printf "WARNINGS=%s\nERRORS=%s\n" "${warnings_count}" "${errors_count}" > build-metrics.env
80+
81+
{
82+
echo "full-build-base warnings: ${warnings_count}"
83+
echo "full-build-base errors: ${errors_count}"
84+
if [ -n "${previous_warnings}" ] && [ -n "${previous_errors}" ]; then
85+
echo "delta warnings vs last successful ${{ inputs.baseline-branch }} run: ${warnings_delta}"
86+
echo "delta errors vs last successful ${{ inputs.baseline-branch }} run: ${errors_delta}"
87+
else
88+
echo "No previous metrics artifact found on ${{ inputs.baseline-branch }} for comparison."
89+
fi
90+
} >> "$GITHUB_STEP_SUMMARY"
91+
92+
if [ "${regression}" = "true" ]; then
93+
echo "METRICS_REGRESSION=true" >> "$GITHUB_ENV"
94+
else
95+
echo "METRICS_REGRESSION=false" >> "$GITHUB_ENV"
96+
fi
97+
98+
- name: Upload current metrics artifact
99+
uses: actions/upload-artifact@v7
100+
with:
101+
name: ${{ inputs.artifact-name }}
102+
path: build-metrics.env
103+
retention-days: ${{ inputs.retention-days }}
104+
if-no-files-found: error
105+
106+
- name: Mark metrics regression
107+
if: ${{ env.METRICS_REGRESSION == 'true' }}
108+
shell: bash
109+
run: exit 1

‎.github/workflows/ci.yml‎

Lines changed: 67 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ env:
4040
# https://github.com/WebFuzzing/EvoMaster/issues/447
4141
release-jdk: 21
4242
build-jdk: 17
43+
java-distribution: temurin
4344
retention-days: 5
4445
debug: false # put to true if need to debug a specific test
4546
debugTestName: "GeneRandomizedTest" # replace with test to debug
@@ -74,14 +75,14 @@ jobs:
7475
needs: setup
7576
if: needs.setup.outputs.debug == 'true'
7677
steps:
77-
- uses: actions/checkout@v4
78+
- uses: actions/checkout@v6
7879
- name: Setup JDK ${{env.build-jdk}}
7980
uses: actions/setup-java@v5
8081
with:
82+
distribution: ${{env.java-distribution}}
8183
java-version: ${{env.build-jdk}}
82-
distribution: 'temurin'
8384
- name: Cache Maven packages
84-
uses: actions/cache@v3
85+
uses: actions/cache@v5
8586
with:
8687
path: ~/.m2
8788
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -97,25 +98,36 @@ jobs:
9798
if: needs.setup.outputs.debug == 'false'
9899
steps:
99100
# Checkout code
100-
- uses: actions/checkout@v4
101+
- uses: actions/checkout@v6
101102
# Build/test for JDK
102103
- name: Setup JDK ${{env.build-jdk}}
103104
uses: actions/setup-java@v5
104105
with:
106+
distribution: ${{env.java-distribution}}
105107
java-version: ${{env.build-jdk}}
106-
distribution: 'temurin'
107108
- name: Cache Maven packages
108-
uses: actions/cache@v3
109+
uses: actions/cache@v5
109110
with:
110111
path: ~/.m2
111112
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
112113
restore-keys: ${{ runner.os }}-m2
113114
- name: Build with Maven
114-
run: mvn clean verify --fae
115+
# the set command makes the build fail if mvn command fails (no change in behavior)
116+
run: |
117+
set -o pipefail
118+
mvn clean verify --fae 2>&1 | tee full-build-base.log
115119
env:
116120
CI_env: GithubAction
121+
- name: Upload full-build-base log for metrics job
122+
if: success()
123+
uses: actions/upload-artifact@v7
124+
with:
125+
name: full-build-base-log-current
126+
path: full-build-base.log
127+
retention-days: 1
128+
if-no-files-found: error
117129
- name: Upload evomaster.jar
118-
uses: actions/upload-artifact@v4
130+
uses: actions/upload-artifact@v7
119131
with:
120132
name: evomaster.jar
121133
path: core/target/evomaster.jar
@@ -140,21 +152,41 @@ jobs:
140152
name: Uploading coverage to CodeCov is done only on 'master' branch builds
141153
run: echo Skipping upload to CodeCov
142154

143-
155+
full-build-metrics:
156+
runs-on: ubuntu-latest
157+
needs: full-build-base
158+
if: needs.full-build-base.result == 'success'
159+
steps:
160+
- uses: actions/checkout@v6
161+
- name: Download full-build-base log
162+
uses: actions/download-artifact@v8
163+
with:
164+
name: full-build-base-log-current
165+
path: .
166+
- name: Analyze and persist full-build-base metrics
167+
continue-on-error: true
168+
uses: ./.github/actions/full-build-metrics
169+
with:
170+
log-file: full-build-base.log
171+
artifact-name: full-build-base-metrics
172+
baseline-branch: master
173+
workflow-file: ci.yml
174+
#These metrics occupy very little space, increasing the retention so they are not lost if nothing is merged to master in a while
175+
retention-days: 90
144176

145177
base-build-mac:
146178
runs-on: macos-latest
147179
needs: setup
148180
if: needs.setup.outputs.debug == 'false'
149181
steps:
150-
- uses: actions/checkout@v4
182+
- uses: actions/checkout@v6
151183
- name: Setup JDK ${{env.release-jdk}}
152184
uses: actions/setup-java@v5
153185
with:
186+
distribution: ${{env.java-distribution}}
154187
java-version: ${{env.release-jdk}}
155-
distribution: 'temurin'
156188
- name: Cache Maven packages
157-
uses: actions/cache@v3
189+
uses: actions/cache@v5
158190
with:
159191
path: ~/.m2
160192
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -169,14 +201,14 @@ jobs:
169201
needs: setup
170202
if: needs.setup.outputs.debug == 'false'
171203
steps:
172-
- uses: actions/checkout@v4
204+
- uses: actions/checkout@v6
173205
- name: Setup JDK ${{env.release-jdk}}
174206
uses: actions/setup-java@v5
175207
with:
208+
distribution: ${{env.java-distribution}}
176209
java-version: ${{env.release-jdk}}
177-
distribution: 'temurin'
178210
- name: Cache Maven packages
179-
uses: actions/cache@v3
211+
uses: actions/cache@v5
180212
with:
181213
path: ~/.m2
182214
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -225,22 +257,22 @@ jobs:
225257
needs: full-build-base
226258
runs-on: windows-latest
227259
steps:
228-
- uses: actions/checkout@v4
260+
- uses: actions/checkout@v6
229261
- name: Setup JDK ${{env.release-jdk}}
230262
uses: actions/setup-java@v5
231263
with:
264+
distribution: ${{env.java-distribution}}
232265
java-version: ${{env.release-jdk}}
233-
distribution: 'temurin'
234266
- name: Download fat jar
235-
uses: actions/download-artifact@v4
267+
uses: actions/download-artifact@v8
236268
with:
237269
name: evomaster.jar
238270
path: core/target
239271
- name: Build installation file
240272
shell: bash
241273
run: bash makeExecutable.sh WINDOWS
242274
- name: Upload installation file
243-
uses: actions/upload-artifact@v4
275+
uses: actions/upload-artifact@v7
244276
with:
245277
name: evomaster.msi
246278
path: release/evomaster-${{env.evomaster-version}}.msi
@@ -251,22 +283,22 @@ jobs:
251283
needs: full-build-base
252284
runs-on: macos-latest
253285
steps:
254-
- uses: actions/checkout@v3
286+
- uses: actions/checkout@v6
255287
- name: Setup JDK ${{env.release-jdk}}
256288
uses: actions/setup-java@v5
257289
with:
290+
distribution: ${{env.java-distribution}}
258291
java-version: ${{env.release-jdk}}
259-
distribution: 'temurin'
260292
- name: Download fat jar
261-
uses: actions/download-artifact@v4
293+
uses: actions/download-artifact@v8
262294
with:
263295
name: evomaster.jar
264296
path: core/target
265297
- name: Build installation file
266298
shell: bash
267299
run: bash makeExecutable.sh OSX
268300
- name: Upload installation file
269-
uses: actions/upload-artifact@v4
301+
uses: actions/upload-artifact@v7
270302
with:
271303
name: evomaster.dmg
272304
path: release/evomaster-${{env.evomaster-version}}.dmg
@@ -277,14 +309,14 @@ jobs:
277309
needs: full-build-base
278310
runs-on: ubuntu-latest
279311
steps:
280-
- uses: actions/checkout@v4
312+
- uses: actions/checkout@v6
281313
- name: Setup JDK ${{env.release-jdk}}
282314
uses: actions/setup-java@v5
283315
with:
316+
distribution: ${{env.java-distribution}}
284317
java-version: ${{env.release-jdk}}
285-
distribution: 'temurin'
286318
- name: Download fat jar
287-
uses: actions/download-artifact@v4
319+
uses: actions/download-artifact@v8
288320
with:
289321
name: evomaster.jar
290322
path: core/target
@@ -295,7 +327,7 @@ jobs:
295327
shell: bash
296328
run: ls -l release
297329
- name: Upload installation file
298-
uses: actions/upload-artifact@v4
330+
uses: actions/upload-artifact@v7
299331
with:
300332
name: evomaster.deb
301333
# JDK 17 and 21 use different suffixes... doesn't seem configurable :(
@@ -308,11 +340,11 @@ jobs:
308340
test-utils-js:
309341
runs-on: ubuntu-latest
310342
steps:
311-
- uses: actions/checkout@v4
343+
- uses: actions/checkout@v6
312344
- name: Use Node.js
313-
uses: actions/setup-node@v4
345+
uses: actions/setup-node@v6
314346
with:
315-
node-version: 20
347+
node-version: 24
316348
- run: npm ci
317349
working-directory: ./test-utils/test-utils-js
318350
- run: npm test
@@ -322,9 +354,9 @@ jobs:
322354
test-utils-py:
323355
runs-on: ubuntu-latest
324356
steps:
325-
- uses: actions/checkout@v4
357+
- uses: actions/checkout@v6
326358
- name: Set up Python
327-
uses: actions/setup-python@v5
359+
uses: actions/setup-python@v6
328360
with:
329361
python-version: '3.10'
330362
- name: Install dependencies
@@ -345,14 +377,14 @@ jobs:
345377
needs: setup
346378
if: needs.setup.outputs.debug == 'false'
347379
steps:
348-
- uses: actions/checkout@v4
380+
- uses: actions/checkout@v6
349381
- name: Setup JDK ${{env.build-jdk}}
350382
uses: actions/setup-java@v5
351383
with:
384+
distribution: ${{env.java-distribution}}
352385
java-version: ${{env.build-jdk}}
353-
distribution: 'temurin'
354386
- name: Cache Maven packages
355-
uses: actions/cache@v3
387+
uses: actions/cache@v5
356388
with:
357389
path: ~/.m2
358390
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

0 commit comments

Comments
 (0)