Skip to content

Commit 138e6b9

Browse files
arcuri82arcuri82
committed
starting with lightweight HTTP status code oracles
1 parent d14a9aa commit 138e6b9

6 files changed

Lines changed: 137 additions & 0 deletions

File tree

‎core/src/main/kotlin/org/evomaster/core/EMConfig.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2957,6 +2957,10 @@ class EMConfig {
29572957
@Cfg("Extra checks on HTTP properties in returned responses, used as automated oracles to detect faults.")
29582958
var httpOracles = false
29592959

2960+
@Experimental
2961+
@Cfg("Lightweight checks on HTTP status codes, e.g., a GET should not return a 201 Created.")
2962+
var statusOracles = false
2963+
29602964
@Cfg("Validate responses against their schema, to check for inconsistencies. Those are treated as faults.")
29612965
var schemaOracles = true
29622966

‎core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,18 @@ enum class ExperimentalFaultCategory(
4141
"TODO"),
4242

4343

44+
HTTP_STATUS_NO_NON_STANDARD_CODES(950, "no-non-standard-codes", "invalidStatusCode", "TODO"),
45+
HTTP_STATUS_NO_201_IF_DELETE(951, "no-201-if-delete", "201OnDelete", "TODO"),
46+
HTTP_STATUS_NO_201_IF_GET(952, "no-201-if-get", "201OnGet", "TODO"),
47+
HTTP_STATUS_NO_201_IF_PATCH(953, "no-201-if-patch", "201OnPatch", "TODO"),
48+
HTTP_STATUS_NO_204_IF_CONTENT(954, "no-204-if-content", "204WhenContent", "TODO"),
49+
HTTP_STATUS_NO_413_IF_NO_PAYLOAD(955, "no-413-if-no-payload", "413WhenNoPayload", "TODO"),
50+
HTTP_STATUS_NO_415_IF_NO_PAYLOAD(956, "no-415-if-no-payload", "415WhenNoPayload", "TODO"),
51+
HTTP_STATUS_NO_401_IF_NO_AUTH(957, "no-401-if-no-auth", "401WhenNoAuth", "TODO"),
52+
HTTP_STATUS_NO_403_IF_NO_401(958, "no-403-if-no-401", "403WhenNo401", "TODO"),
53+
HTTP_STATUS_HAS_406_IF_ACCEPT(959, "has-406-if-accept", "406WhenValid", "TODO"),
54+
55+
4456
//3xx: GraphQL
4557
GQL_ERROR_FIELD(920, "Error Field", "returnedErrors",
4658
"TODO"),

‎core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpStatusOracle.kt‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package org.evomaster.core.problem.rest.oracle
2+
3+
import com.webfuzzing.commons.faults.FaultCategory
4+
import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory
5+
import org.evomaster.core.problem.rest.data.HttpVerb
6+
import org.evomaster.core.problem.rest.data.RestCallAction
7+
import org.evomaster.core.problem.rest.data.RestCallResult
8+
import org.evomaster.core.problem.rest.param.BodyParam
9+
10+
/**
11+
* Series of oracles that can be evaluated directly on the HTTP responses based on the actual retrieved status code.
12+
* Some checks might depend on the API schema and/or the input request.
13+
* These checks are based on work done in:
14+
*
15+
* https://github.com/alixdecr/scoas
16+
* https://github.com/alixdecr/scoas/blob/main/evaluation/status-code-rules.md
17+
*
18+
*
19+
* Not all oracles can be translated to a dynamic version. Here, we just support the following:
20+
* no-non-standard-codes
21+
* no-201-if-delete
22+
* no-201-if-get
23+
* no-201-if-patch
24+
* no-204-if-content
25+
* no-413-if-no-payload
26+
* no-415-if-no-payload
27+
* no-401-if-no-auth (schema)
28+
* no-403-if-no-401 (schema)
29+
* has-406-if-accept (schema)
30+
*
31+
*
32+
* IMPORTANT: in contrast to what done in [HttpSemanticsOracle], here there is no need to construct any test case
33+
* on purpose with specific properties.
34+
* These checks are supposed to be lightweight, and so could be checked on each fitness evaluation.
35+
*/
36+
object HttpStatusOracle {
37+
38+
39+
fun checkOracles(call: RestCallAction, result: RestCallResult) : List<FaultCategory>{
40+
41+
val faults = mutableListOf<FaultCategory>()
42+
43+
val status = result.getStatusCode()
44+
?: return faults // all oracles depend on checking the status code
45+
46+
if(status !in 100..599){
47+
faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_NON_STANDARD_CODES)
48+
}
49+
50+
val verb = call.verb
51+
52+
if(status == 201){
53+
when(verb){
54+
HttpVerb.GET -> faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_201_IF_GET)
55+
HttpVerb.DELETE -> faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_201_IF_DELETE)
56+
HttpVerb.PATCH -> faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_201_IF_PATCH)
57+
else -> {}
58+
}
59+
}
60+
61+
if(status == 204 && verb == HttpVerb.GET){
62+
faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_204_IF_CONTENT)
63+
}
64+
65+
val bodyParam = call.parameters.filterIsInstance<BodyParam>()
66+
.firstOrNull()
67+
68+
val hasBody = bodyParam != null && bodyParam.primaryGene().getValueAsRawString().isNotEmpty()
69+
70+
if(status == 413 && !hasBody){
71+
faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_413_IF_NO_PAYLOAD)
72+
}
73+
74+
if(status == 415 && !hasBody){
75+
faults.add(ExperimentalFaultCategory.HTTP_STATUS_NO_415_IF_NO_PAYLOAD)
76+
}
77+
78+
if(status == 406 && !call.isForRobustnessTesting()){
79+
faults.add(ExperimentalFaultCategory.HTTP_STATUS_HAS_406_IF_ACCEPT)
80+
}
81+
82+
if(status == 401){
83+
//TODO
84+
}
85+
86+
if(status == 403){
87+
//TODO
88+
}
89+
90+
return faults
91+
}
92+
}

‎core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import org.evomaster.core.problem.rest.data.RestCallResult
2525
import org.evomaster.core.problem.rest.data.RestIndividual
2626
import org.evomaster.core.problem.rest.link.RestLinkValueUpdater
2727
import org.evomaster.core.problem.rest.oracle.HttpSemanticsOracle
28+
import org.evomaster.core.problem.rest.oracle.HttpStatusOracle
2829
import org.evomaster.core.problem.rest.oracle.RestSchemaOracle
2930
import org.evomaster.core.problem.rest.oracle.RestSecurityOracle
3031
import org.evomaster.core.problem.rest.param.BodyParam
@@ -386,13 +387,31 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
386387
fv.updateTarget(statusId, 1.0, it)
387388

388389
handleAdvancedBlackBoxCriteria(fv, actions[it], result)
390+
handleHttpStatusOracles(fv, actions[it], result, it)
389391

390392
val location5xx: String? = getlocation5xx(status, additionalInfoList, it, result, name)
391393
handleAdditionalStatusTargetDescription(result, fv, status, name, it, location5xx)
392394
handleAuthTargets(status, actions, it, name, fv)
393395
}
394396
}
395397

398+
private fun handleHttpStatusOracles(fv: FitnessValue, call: RestCallAction, result: RestCallResult, index: Int) {
399+
if(!config.statusOracles){
400+
return
401+
}
402+
403+
val name = call.getName()
404+
405+
//TODO schema input
406+
HttpStatusOracle.checkOracles(call, result)
407+
.filter { config.isEnabledFaultCategory(it) }
408+
.forEach {
409+
val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(it, name))
410+
fv.updateTarget(scenarioId, 1.0, index)
411+
result.addFault(DetectedFault(it, name, null))
412+
}
413+
}
414+
396415
private fun handleAuthTargets(
397416
status: Int,
398417
actions: List<RestCallAction>,

‎core/src/main/kotlin/org/evomaster/core/search/action/Action.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ abstract class Action(children: List<StructuralElement>) : ActionComponent(
4141
*/
4242
abstract fun seeTopGenes(): List<Gene>
4343

44+
/**
45+
* Check if this action has invalid data on purpose, done or Robustness Testing
46+
*/
47+
open fun isForRobustnessTesting() : Boolean{
48+
//TODO currently this is just a place-holder, needed for code that we already know will
49+
//depend on this check
50+
return false
51+
}
52+
4453
fun seeAllGenes(): List<Gene> = seeTopGenes().flatMap {g -> g.flatView() }
4554

4655
final override fun copy(): Action {

‎docs/options.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ There are 3 types of options:
328328
|`sqliBaselineMaxResponseTimeMs`| __Int__. Maximum allowed baseline response time (in milliseconds) before the malicious payload is applied. *Depends on*: `sqli=true`. *Default value*: `2000`.|
329329
|`sqliInjectedSleepDurationMs`| __Int__. Injected sleep duration (in seconds) used inside the malicious payload to detect time-based vulnerabilities. *Depends on*: `sqli=true`. *Default value*: `5000`.|
330330
|`ssrf`| __Boolean__. To apply SSRF detection as part of security testing. *Depends on*: `security=true`. *Default value*: `false`.|
331+
|`statusOracles`| __Boolean__. Lightweight checks on HTTP status codes, e.g., a GET should not return a 201 Created. *Default value*: `false`.|
331332
|`structureMutationProFS`| __Double__. Specify a probability of applying structure mutator during the focused search. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
332333
|`structureMutationProbStrategy`| __Enum__. Specify a strategy to handle a probability of applying structure mutator during the focused search. *Valid values*: `SPECIFIED, SPECIFIED_FS, DPC_TO_SPECIFIED_BEFORE_FS, DPC_TO_SPECIFIED_AFTER_FS, ADAPTIVE_WITH_IMPACT`. *Default value*: `SPECIFIED`.|
333334
|`sutDistEnvVarName`| __String__. Specify name of the environment variable that provides the the base distribution directory of the SUT, e.g., 'dist' directory of WFD. *Default value*: `""`.|

0 commit comments

Comments
 (0)