Skip to content

Commit 9946b9a

Browse files
BryceWDesignBryceWDesign
authored
Add unit tests for IX cognition negative examples
1 parent b661361 commit 9946b9a

1 file changed

Lines changed: 243 additions & 0 deletions

File tree

‎tests/test_ix_cognition_negative_examples.py‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
import io
2+
import json
3+
import unittest
4+
from contextlib import redirect_stdout
5+
from pathlib import Path
6+
7+
from ix.assurance import assess_ix
8+
from ix.cli import main
9+
from ix.formatting import format_ix
10+
from ix.parser import parse_ix
11+
12+
13+
REPO_ROOT = Path(__file__).resolve().parents[1]
14+
EXAMPLES_DIR = REPO_ROOT / "examples"
15+
16+
NEGATIVE_EXAMPLES = {
17+
"cognitionkernel_wave6_missing_obligations.ix": {
18+
"expected_checks": {
19+
"cognition_contract.required_obligations.missing",
20+
},
21+
"expected_missing": {
22+
"purpose_discipline",
23+
"claim_boundary_discipline",
24+
"human_authority",
25+
"reality_delta_comparison",
26+
"evidence_bound_memory_update",
27+
"future_reasoning_change",
28+
"cross_domain_transfer_probe",
29+
"novelty_generality_pressure",
30+
"long_horizon_planning_trace",
31+
"uncertainty_assumption_exposure",
32+
"contradiction_handling",
33+
"shortcut_reward_hacking_detection",
34+
"safe_refusal_path",
35+
"self_improvement_airlock",
36+
"no_self_certification",
37+
"falsification_ledger",
38+
"independent_replay_review",
39+
"kernel_handoff_package",
40+
},
41+
},
42+
"cognitionkernel_wave6_overclaiming.ix": {
43+
"expected_checks": {
44+
"cognition_contract.prohibited_claim_language.present",
45+
"cognition_contract.required_obligations.missing",
46+
},
47+
"expected_missing": {
48+
"purpose_discipline",
49+
"claim_boundary_discipline",
50+
"human_authority",
51+
"measured_outcome_capture",
52+
"reality_delta_comparison",
53+
"evidence_bound_memory_update",
54+
"future_reasoning_change",
55+
"cross_domain_transfer_probe",
56+
"novelty_generality_pressure",
57+
"long_horizon_planning_trace",
58+
"uncertainty_assumption_exposure",
59+
"contradiction_handling",
60+
"shortcut_reward_hacking_detection",
61+
"safe_refusal_path",
62+
"self_improvement_airlock",
63+
"no_self_certification",
64+
"falsification_ledger",
65+
"independent_replay_review",
66+
"kernel_handoff_package",
67+
},
68+
},
69+
"cognitionkernel_wave6_wrong_handoff.ix": {
70+
"expected_checks": {
71+
"cognition_contract.kernel_handoff.missing",
72+
"cognition_contract.required_obligations.missing",
73+
},
74+
"expected_missing": {
75+
"purpose_discipline",
76+
"claim_boundary_discipline",
77+
"human_authority",
78+
"measured_outcome_capture",
79+
"reality_delta_comparison",
80+
"evidence_bound_memory_update",
81+
"future_reasoning_change",
82+
"cross_domain_transfer_probe",
83+
"novelty_generality_pressure",
84+
"long_horizon_planning_trace",
85+
"uncertainty_assumption_exposure",
86+
"contradiction_handling",
87+
"shortcut_reward_hacking_detection",
88+
"safe_refusal_path",
89+
"self_improvement_airlock",
90+
"no_self_certification",
91+
"falsification_ledger",
92+
"independent_replay_review",
93+
"kernel_handoff_package",
94+
},
95+
},
96+
"cognitionkernel_wave6_noncanonical_falsification.ix": {
97+
"expected_checks": {
98+
"cognition_contract.obligation_canonical_falsification.missing",
99+
"cognition_contract.required_obligations.missing",
100+
},
101+
"expected_missing": {
102+
"purpose_discipline",
103+
"claim_boundary_discipline",
104+
"human_authority",
105+
"measured_outcome_capture",
106+
"reality_delta_comparison",
107+
"evidence_bound_memory_update",
108+
"future_reasoning_change",
109+
"cross_domain_transfer_probe",
110+
"novelty_generality_pressure",
111+
"long_horizon_planning_trace",
112+
"uncertainty_assumption_exposure",
113+
"contradiction_handling",
114+
"shortcut_reward_hacking_detection",
115+
"safe_refusal_path",
116+
"self_improvement_airlock",
117+
"no_self_certification",
118+
"falsification_ledger",
119+
"independent_replay_review",
120+
"kernel_handoff_package",
121+
},
122+
},
123+
}
124+
125+
126+
class TestIXCognitionNegativeExamples(unittest.TestCase):
127+
def test_negative_examples_parse_and_format_cleanly(self):
128+
for example_name in NEGATIVE_EXAMPLES:
129+
with self.subTest(example=example_name):
130+
example_file = EXAMPLES_DIR / example_name
131+
source = example_file.read_text(encoding="utf-8")
132+
program = parse_ix(source, filename=str(example_file))
133+
134+
self.assertEqual(format_ix(program), source)
135+
136+
def test_negative_examples_fail_cognitionkernel_wave6_profile(self):
137+
for example_name, expectation in NEGATIVE_EXAMPLES.items():
138+
with self.subTest(example=example_name):
139+
example_file = EXAMPLES_DIR / example_name
140+
program = parse_ix(
141+
example_file.read_text(encoding="utf-8"),
142+
filename=str(example_file),
143+
)
144+
145+
report = assess_ix(program, profile="cognitionkernel-wave6")
146+
check_ids = {check.check_id for check in report.checks}
147+
148+
self.assertEqual(report.status, "fail")
149+
self.assertTrue(expectation["expected_checks"].issubset(check_ids))
150+
151+
def test_negative_examples_report_expected_missing_obligations(self):
152+
for example_name, expectation in NEGATIVE_EXAMPLES.items():
153+
with self.subTest(example=example_name):
154+
example_file = EXAMPLES_DIR / example_name
155+
program = parse_ix(
156+
example_file.read_text(encoding="utf-8"),
157+
filename=str(example_file),
158+
)
159+
160+
report = assess_ix(program, profile="cognitionkernel-wave6")
161+
missing_checks = [
162+
check
163+
for check in report.checks
164+
if check.check_id
165+
== "cognition_contract.required_obligations.missing"
166+
]
167+
168+
self.assertEqual(len(missing_checks), 1)
169+
self.assertEqual(
170+
set(missing_checks[0].data["missing_obligations"]),
171+
expectation["expected_missing"],
172+
)
173+
174+
def test_overclaiming_example_reports_prohibited_claim_text(self):
175+
example_file = EXAMPLES_DIR / "cognitionkernel_wave6_overclaiming.ix"
176+
program = parse_ix(
177+
example_file.read_text(encoding="utf-8"),
178+
filename=str(example_file),
179+
)
180+
181+
report = assess_ix(program, profile="cognitionkernel-wave6")
182+
prohibited_checks = [
183+
check
184+
for check in report.checks
185+
if check.check_id == "cognition_contract.prohibited_claim_language.present"
186+
]
187+
188+
self.assertEqual(len(prohibited_checks), 1)
189+
self.assertIn(
190+
"this contract proves agi achieved",
191+
prohibited_checks[0].data["matched_text"],
192+
)
193+
194+
def test_noncanonical_falsification_example_reports_gate_mismatch(self):
195+
example_file = EXAMPLES_DIR / "cognitionkernel_wave6_noncanonical_falsification.ix"
196+
program = parse_ix(
197+
example_file.read_text(encoding="utf-8"),
198+
filename=str(example_file),
199+
)
200+
201+
report = assess_ix(program, profile="cognitionkernel-wave6")
202+
missing_checks = [
203+
check
204+
for check in report.checks
205+
if check.check_id
206+
== "cognition_contract.obligation_canonical_falsification.missing"
207+
]
208+
209+
self.assertEqual(len(missing_checks), 1)
210+
self.assertEqual(missing_checks[0].data["obligation"], "prediction_before_trial")
211+
self.assertEqual(
212+
missing_checks[0].data["declared_conditions"],
213+
["arbitrary_failure_label"],
214+
)
215+
self.assertIn("prediction_missing", missing_checks[0].data["canonical_conditions"])
216+
217+
def test_negative_examples_fail_closed_through_cli_assure(self):
218+
for example_name, expectation in NEGATIVE_EXAMPLES.items():
219+
with self.subTest(example=example_name):
220+
example_file = EXAMPLES_DIR / example_name
221+
stdout = io.StringIO()
222+
223+
with redirect_stdout(stdout):
224+
code = main(
225+
[
226+
"assure",
227+
str(example_file),
228+
"--profile",
229+
"cognitionkernel-wave6",
230+
"--json",
231+
]
232+
)
233+
234+
self.assertEqual(code, 2)
235+
payload = json.loads(stdout.getvalue())
236+
check_ids = {check["id"] for check in payload["checks"]}
237+
238+
self.assertEqual(payload["status"], "fail")
239+
self.assertTrue(expectation["expected_checks"].issubset(check_ids))
240+
241+
242+
if __name__ == "__main__":
243+
unittest.main()

0 commit comments

Comments
 (0)