Skip to content

Commit af54905

Browse files
omursahinomursahin
authored
Merge branch 'master' into oracle-partial-update-put
2 parents c09bdff + d14a9aa commit af54905

360 files changed

Lines changed: 9040 additions & 1441 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/full-build-metrics/action.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: Full Build Metrics
2+
description: Count Maven warnings/errors, compare against latest successful baseline, and persist current metrics artifact
3+
4+
inputs:
5+
log-file:
6+
description: Path to the captured build log
7+
required: false
8+
default: full-build-base.log
9+
artifact-name:
10+
description: Artifact name used to store metrics
11+
required: false
12+
default: full-build-base-metrics
13+
baseline-branch:
14+
description: Branch to fetch the latest successful baseline metrics from
15+
required: false
16+
default: master
17+
workflow-file:
18+
description: Workflow filename to query for baseline artifacts
19+
required: false
20+
default: ci.yml
21+
retention-days:
22+
description: Artifact retention period in days
23+
required: false
24+
default: "90"
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Download previous metrics artifact
30+
uses: dawidd6/action-download-artifact@v20
31+
with:
32+
workflow: ${{ inputs.workflow-file }}
33+
workflow_conclusion: success
34+
branch: ${{ inputs.baseline-branch }}
35+
name: ${{ inputs.artifact-name }}
36+
path: previous-metrics
37+
if_no_artifact_found: warn
38+
39+
- name: Count warnings/errors and compare
40+
shell: bash
41+
run: |
42+
if [ -f "${{ inputs.log-file }}" ]; then
43+
warnings_count=$(grep -c "^\[WARNING\]" "${{ inputs.log-file }}" || true)
44+
errors_count=$(grep -c "^\[ERROR\]" "${{ inputs.log-file }}" || true)
45+
else
46+
warnings_count=9999999
47+
errors_count=9999999
48+
fi
49+
50+
previous_warnings=""
51+
previous_errors=""
52+
if [ -f previous-metrics/build-metrics.env ]; then
53+
# shellcheck disable=SC1091
54+
. previous-metrics/build-metrics.env
55+
previous_warnings="${WARNINGS:-}"
56+
previous_errors="${ERRORS:-}"
57+
fi
58+
59+
echo "full-build-base warnings: ${warnings_count}"
60+
echo "full-build-base errors: ${errors_count}"
61+
62+
regression=false
63+
if [ -n "${previous_warnings}" ] && [ -n "${previous_errors}" ]; then
64+
warnings_delta=$((warnings_count - previous_warnings))
65+
errors_delta=$((errors_count - previous_errors))
66+
echo "delta warnings vs last successful ${{ inputs.baseline-branch }} run: ${warnings_delta}"
67+
echo "delta errors vs last successful ${{ inputs.baseline-branch }} run: ${errors_delta}"
68+
if [ "${warnings_count}" -le "${previous_warnings}" ] && [ "${errors_count}" -le "${previous_errors}" ]; then
69+
echo "Result: green (no increase in warnings or errors)."
70+
else
71+
echo "Result: yellow (warnings or errors increased)."
72+
regression=true
73+
fi
74+
else
75+
echo "No previous metrics artifact found on ${{ inputs.baseline-branch }} for comparison."
76+
echo "Result: green (no baseline available)."
77+
fi
78+
79+
printf "WARNINGS=%s\nERRORS=%s\n" "${warnings_count}" "${errors_count}" > build-metrics.env
80+
81+
{
82+
echo "full-build-base warnings: ${warnings_count}"
83+
echo "full-build-base errors: ${errors_count}"
84+
if [ -n "${previous_warnings}" ] && [ -n "${previous_errors}" ]; then
85+
echo "delta warnings vs last successful ${{ inputs.baseline-branch }} run: ${warnings_delta}"
86+
echo "delta errors vs last successful ${{ inputs.baseline-branch }} run: ${errors_delta}"
87+
else
88+
echo "No previous metrics artifact found on ${{ inputs.baseline-branch }} for comparison."
89+
fi
90+
} >> "$GITHUB_STEP_SUMMARY"
91+
92+
if [ "${regression}" = "true" ]; then
93+
echo "METRICS_REGRESSION=true" >> "$GITHUB_ENV"
94+
else
95+
echo "METRICS_REGRESSION=false" >> "$GITHUB_ENV"
96+
fi
97+
98+
- name: Upload current metrics artifact
99+
uses: actions/upload-artifact@v7
100+
with:
101+
name: ${{ inputs.artifact-name }}
102+
path: build-metrics.env
103+
retention-days: ${{ inputs.retention-days }}
104+
if-no-files-found: error
105+
106+
- name: Mark metrics regression
107+
if: ${{ env.METRICS_REGRESSION == 'true' }}
108+
shell: bash
109+
run: exit 1

‎.github/workflows/ci.yml‎

Lines changed: 77 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ env:
4040
# https://github.com/WebFuzzing/EvoMaster/issues/447
4141
release-jdk: 21
4242
build-jdk: 17
43+
java-distribution: temurin
4344
retention-days: 5
4445
debug: false # put to true if need to debug a specific test
4546
debugTestName: "GeneRandomizedTest" # replace with test to debug
@@ -74,13 +75,14 @@ jobs:
7475
needs: setup
7576
if: needs.setup.outputs.debug == 'true'
7677
steps:
77-
- uses: actions/checkout@v4
78+
- uses: actions/checkout@v6
7879
- name: Setup JDK ${{env.build-jdk}}
79-
uses: actions/setup-java@v1
80+
uses: actions/setup-java@v5
8081
with:
82+
distribution: ${{env.java-distribution}}
8183
java-version: ${{env.build-jdk}}
8284
- name: Cache Maven packages
83-
uses: actions/cache@v3
85+
uses: actions/cache@v5
8486
with:
8587
path: ~/.m2
8688
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -96,24 +98,36 @@ jobs:
9698
if: needs.setup.outputs.debug == 'false'
9799
steps:
98100
# Checkout code
99-
- uses: actions/checkout@v4
101+
- uses: actions/checkout@v6
100102
# Build/test for JDK
101103
- name: Setup JDK ${{env.build-jdk}}
102-
uses: actions/setup-java@v1
104+
uses: actions/setup-java@v5
103105
with:
106+
distribution: ${{env.java-distribution}}
104107
java-version: ${{env.build-jdk}}
105108
- name: Cache Maven packages
106-
uses: actions/cache@v3
109+
uses: actions/cache@v5
107110
with:
108111
path: ~/.m2
109112
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
110113
restore-keys: ${{ runner.os }}-m2
111114
- name: Build with Maven
112-
run: mvn clean verify --fae
115+
# the set command makes the build fail if mvn command fails (no change in behavior)
116+
run: |
117+
set -o pipefail
118+
mvn clean verify --fae 2>&1 | tee full-build-base.log
113119
env:
114120
CI_env: GithubAction
121+
- name: Upload full-build-base log for metrics job
122+
if: success()
123+
uses: actions/upload-artifact@v7
124+
with:
125+
name: full-build-base-log-current
126+
path: full-build-base.log
127+
retention-days: 1
128+
if-no-files-found: error
115129
- name: Upload evomaster.jar
116-
uses: actions/upload-artifact@v4
130+
uses: actions/upload-artifact@v7
117131
with:
118132
name: evomaster.jar
119133
path: core/target/evomaster.jar
@@ -138,20 +152,41 @@ jobs:
138152
name: Uploading coverage to CodeCov is done only on 'master' branch builds
139153
run: echo Skipping upload to CodeCov
140154

141-
155+
full-build-metrics:
156+
runs-on: ubuntu-latest
157+
needs: full-build-base
158+
if: needs.full-build-base.result == 'success'
159+
steps:
160+
- uses: actions/checkout@v6
161+
- name: Download full-build-base log
162+
uses: actions/download-artifact@v8
163+
with:
164+
name: full-build-base-log-current
165+
path: .
166+
- name: Analyze and persist full-build-base metrics
167+
continue-on-error: true
168+
uses: ./.github/actions/full-build-metrics
169+
with:
170+
log-file: full-build-base.log
171+
artifact-name: full-build-base-metrics
172+
baseline-branch: master
173+
workflow-file: ci.yml
174+
#These metrics occupy very little space, increasing the retention so they are not lost if nothing is merged to master in a while
175+
retention-days: 90
142176

143177
base-build-mac:
144178
runs-on: macos-latest
145179
needs: setup
146180
if: needs.setup.outputs.debug == 'false'
147181
steps:
148-
- uses: actions/checkout@v4
182+
- uses: actions/checkout@v6
149183
- name: Setup JDK ${{env.release-jdk}}
150-
uses: actions/setup-java@v1
184+
uses: actions/setup-java@v5
151185
with:
186+
distribution: ${{env.java-distribution}}
152187
java-version: ${{env.release-jdk}}
153188
- name: Cache Maven packages
154-
uses: actions/cache@v3
189+
uses: actions/cache@v5
155190
with:
156191
path: ~/.m2
157192
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -166,13 +201,14 @@ jobs:
166201
needs: setup
167202
if: needs.setup.outputs.debug == 'false'
168203
steps:
169-
- uses: actions/checkout@v4
204+
- uses: actions/checkout@v6
170205
- name: Setup JDK ${{env.release-jdk}}
171-
uses: actions/setup-java@v1
206+
uses: actions/setup-java@v5
172207
with:
208+
distribution: ${{env.java-distribution}}
173209
java-version: ${{env.release-jdk}}
174210
- name: Cache Maven packages
175-
uses: actions/cache@v3
211+
uses: actions/cache@v5
176212
with:
177213
path: ~/.m2
178214
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -196,9 +232,10 @@ jobs:
196232
# steps:
197233
# - uses: actions/checkout@v4
198234
# - name: Setup JDK ${{env.latest-jdk}}
199-
# uses: actions/setup-java@v1
235+
# uses: actions/setup-java@v5
200236
# with:
201237
# java-version: ${{env.latest-jdk}}
238+
# distribution: 'temurin'
202239
# - name: Cache Maven packages
203240
# uses: actions/cache@v3
204241
# with:
@@ -220,21 +257,22 @@ jobs:
220257
needs: full-build-base
221258
runs-on: windows-latest
222259
steps:
223-
- uses: actions/checkout@v4
260+
- uses: actions/checkout@v6
224261
- name: Setup JDK ${{env.release-jdk}}
225-
uses: actions/setup-java@v1
262+
uses: actions/setup-java@v5
226263
with:
264+
distribution: ${{env.java-distribution}}
227265
java-version: ${{env.release-jdk}}
228266
- name: Download fat jar
229-
uses: actions/download-artifact@v4
267+
uses: actions/download-artifact@v8
230268
with:
231269
name: evomaster.jar
232270
path: core/target
233271
- name: Build installation file
234272
shell: bash
235273
run: bash makeExecutable.sh WINDOWS
236274
- name: Upload installation file
237-
uses: actions/upload-artifact@v4
275+
uses: actions/upload-artifact@v7
238276
with:
239277
name: evomaster.msi
240278
path: release/evomaster-${{env.evomaster-version}}.msi
@@ -245,21 +283,22 @@ jobs:
245283
needs: full-build-base
246284
runs-on: macos-latest
247285
steps:
248-
- uses: actions/checkout@v3
286+
- uses: actions/checkout@v6
249287
- name: Setup JDK ${{env.release-jdk}}
250-
uses: actions/setup-java@v1
288+
uses: actions/setup-java@v5
251289
with:
290+
distribution: ${{env.java-distribution}}
252291
java-version: ${{env.release-jdk}}
253292
- name: Download fat jar
254-
uses: actions/download-artifact@v4
293+
uses: actions/download-artifact@v8
255294
with:
256295
name: evomaster.jar
257296
path: core/target
258297
- name: Build installation file
259298
shell: bash
260299
run: bash makeExecutable.sh OSX
261300
- name: Upload installation file
262-
uses: actions/upload-artifact@v4
301+
uses: actions/upload-artifact@v7
263302
with:
264303
name: evomaster.dmg
265304
path: release/evomaster-${{env.evomaster-version}}.dmg
@@ -270,13 +309,14 @@ jobs:
270309
needs: full-build-base
271310
runs-on: ubuntu-latest
272311
steps:
273-
- uses: actions/checkout@v4
312+
- uses: actions/checkout@v6
274313
- name: Setup JDK ${{env.release-jdk}}
275-
uses: actions/setup-java@v1
314+
uses: actions/setup-java@v5
276315
with:
316+
distribution: ${{env.java-distribution}}
277317
java-version: ${{env.release-jdk}}
278318
- name: Download fat jar
279-
uses: actions/download-artifact@v4
319+
uses: actions/download-artifact@v8
280320
with:
281321
name: evomaster.jar
282322
path: core/target
@@ -287,7 +327,7 @@ jobs:
287327
shell: bash
288328
run: ls -l release
289329
- name: Upload installation file
290-
uses: actions/upload-artifact@v4
330+
uses: actions/upload-artifact@v7
291331
with:
292332
name: evomaster.deb
293333
# JDK 17 and 21 use different suffixes... doesn't seem configurable :(
@@ -300,11 +340,11 @@ jobs:
300340
test-utils-js:
301341
runs-on: ubuntu-latest
302342
steps:
303-
- uses: actions/checkout@v4
343+
- uses: actions/checkout@v6
304344
- name: Use Node.js
305-
uses: actions/setup-node@v4
345+
uses: actions/setup-node@v6
306346
with:
307-
node-version: 20
347+
node-version: 24
308348
- run: npm ci
309349
working-directory: ./test-utils/test-utils-js
310350
- run: npm test
@@ -314,9 +354,9 @@ jobs:
314354
test-utils-py:
315355
runs-on: ubuntu-latest
316356
steps:
317-
- uses: actions/checkout@v4
357+
- uses: actions/checkout@v6
318358
- name: Set up Python
319-
uses: actions/setup-python@v5
359+
uses: actions/setup-python@v6
320360
with:
321361
python-version: '3.10'
322362
- name: Install dependencies
@@ -337,13 +377,14 @@ jobs:
337377
needs: setup
338378
if: needs.setup.outputs.debug == 'false'
339379
steps:
340-
- uses: actions/checkout@v4
380+
- uses: actions/checkout@v6
341381
- name: Setup JDK ${{env.build-jdk}}
342-
uses: actions/setup-java@v1
382+
uses: actions/setup-java@v5
343383
with:
384+
distribution: ${{env.java-distribution}}
344385
java-version: ${{env.build-jdk}}
345386
- name: Cache Maven packages
346-
uses: actions/cache@v3
387+
uses: actions/cache@v5
347388
with:
348389
path: ~/.m2
349390
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

0 commit comments

Comments
 (0)