Skip to content

Commit 313c3a5

Browse files
alexeygrigorevalexeygrigorev
committed
Merge issue-206-slice2-repair: Refs #206 anchor portable exports to one instant
2 parents a7291d3 + ec0c62c commit 313c3a5

5 files changed

Lines changed: 322 additions & 32 deletions

File tree

‎backend/src/export/portable.ts‎

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ const VALID_INTAKE_DATA_CLASSES = new Set(['public', 'internal', 'private', 'sen
145145
const VALID_INTAKE_ASSISTANT_STATUSES = new Set(['not-applicable', 'candidate', 'ready', 'submitted', 'blocked']);
146146
const SECRET_EXPORT_PATTERN = /(secret|token|password|credential|cookie|authorization|signed[_-]?url|api[_-]?key)/i;
147147
const SIGNED_URL_EXPORT_PATTERN = /(X-Amz-Signature|X-Amz-Credential|X-Amz-Security-Token|signature=|sig=|access_token=|token=|password=|secret=|credential=|api[_-]?key=)/i;
148+
const RFC3339_INSTANT_PATTERN = /^(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})[Tt](?<hour>\d{2}):(?<minute>\d{2}):(?<second>\d{2})(?:\.(?<fraction>\d{1,3}))?(?:(?<utcZone>[Zz])|(?<offsetSign>[+-])(?<offsetHours>\d{2}):(?<offsetMinutes>\d{2}))$/;
148149

149150
const ENTITY_SPECS: EntitySpec[] = [
150151
{
@@ -605,6 +606,13 @@ function sha256(content: string): string {
605606
return `sha256:${crypto.createHash('sha256').update(content).digest('hex')}`;
606607
}
607608

609+
let portableExportClock: () => Date = () => new Date();
610+
611+
function resolveGeneratedAt(value?: string): string {
612+
if (value === undefined) return portableExportClock().toISOString();
613+
return canonicalizeRfc3339Instant(value);
614+
}
615+
608616
async function writePortableExport(
609617
client: DynamoDBDocumentClient,
610618
outputDir: string,
@@ -616,6 +624,7 @@ async function writePortableExport(
616624
generatedAt?: string;
617625
} = {}
618626
): Promise<PortableExportResult> {
627+
const generatedAt = resolveGeneratedAt(options.generatedAt);
619628
await fs.mkdir(outputDir, { recursive: true });
620629

621630
const entityFiles: Record<string, string> = {};
@@ -624,7 +633,6 @@ async function writePortableExport(
624633

625634
for (const spec of ENTITY_SPECS) {
626635
const rawItems = await scanByPrefix(client, spec.tableName, spec.prefix, spec.recordType, spec.prefixes);
627-
const generatedAt = options.generatedAt || new Date().toISOString();
628636
const records = rawItems
629637
.filter((item) => (
630638
(!spec.filter || spec.filter(item))
@@ -648,7 +656,7 @@ async function writePortableExport(
648656

649657
const manifest: Manifest = {
650658
schema_version: SCHEMA_VERSION,
651-
generated_at: options.generatedAt || new Date().toISOString(),
659+
generated_at: generatedAt,
652660
source_environment: options.sourceEnvironment || process.env.DATAOPS_ENV || process.env.NODE_ENV || 'unknown',
653661
source_stack: options.sourceStack || process.env.AWS_STACK_NAME || 'unknown',
654662
source_region: options.sourceRegion || process.env.AWS_REGION || 'unknown',
@@ -1155,10 +1163,68 @@ function isIsoDate(value: string): boolean {
11551163
);
11561164
}
11571165

1166+
function isRfc3339Instant(value: string, requireUtc = false): boolean {
1167+
const match = RFC3339_INSTANT_PATTERN.exec(value);
1168+
if (!match?.groups) return false;
1169+
1170+
if (requireUtc && !match.groups.utcZone) return false;
1171+
if (Number(match.groups.second) > 59) return false;
1172+
if (match.groups.offsetHours && Number(match.groups.offsetHours) > 23) return false;
1173+
if (match.groups.offsetMinutes && Number(match.groups.offsetMinutes) > 59) return false;
1174+
1175+
// Reapplying the numeric offset recovers the supplied wall-clock fields and
1176+
// prevents JavaScript from rolling invalid dates such as February 30 forward.
1177+
const secondTimestamp = Date.parse(rfc3339SecondBase(value));
1178+
if (Number.isNaN(secondTimestamp)) return false;
1179+
const offsetSign = match.groups.offsetSign === '-' ? -1 : 1;
1180+
const offsetMinutes = (
1181+
Number(match.groups.offsetHours || 0) * 60
1182+
+ Number(match.groups.offsetMinutes || 0)
1183+
) * offsetSign;
1184+
const wallClock = new Date(secondTimestamp + offsetMinutes * 60_000);
1185+
return (
1186+
wallClock.getUTCFullYear() === Number(match.groups.year)
1187+
&& wallClock.getUTCMonth() + 1 === Number(match.groups.month)
1188+
&& wallClock.getUTCDate() === Number(match.groups.day)
1189+
&& wallClock.getUTCHours() === Number(match.groups.hour)
1190+
&& wallClock.getUTCMinutes() === Number(match.groups.minute)
1191+
&& wallClock.getUTCSeconds() === Number(match.groups.second)
1192+
);
1193+
}
1194+
1195+
function rfc3339SecondBase(value: string): string {
1196+
const match = RFC3339_INSTANT_PATTERN.exec(value);
1197+
if (!match?.groups) throw new Error('Value is not an RFC3339 instant');
1198+
1199+
const zone = match.groups.utcZone
1200+
? 'Z'
1201+
: `${match.groups.offsetSign}${match.groups.offsetHours}:${match.groups.offsetMinutes}`;
1202+
return `${match.groups.year}-${match.groups.month}-${match.groups.day}`
1203+
+ `T${match.groups.hour}:${match.groups.minute}:${match.groups.second}.000${zone}`;
1204+
}
1205+
1206+
function canonicalizeRfc3339Instant(value: string): string {
1207+
if (!isRfc3339Instant(value)) {
1208+
throw new Error('generatedAt must be an RFC3339 date-time with millisecond precision');
1209+
}
1210+
1211+
const match = RFC3339_INSTANT_PATTERN.exec(value);
1212+
if (!match?.groups) throw new Error('generatedAt must be an RFC3339 date-time with millisecond precision');
1213+
const utcSeconds = new Date(Date.parse(rfc3339SecondBase(value)))
1214+
.toISOString()
1215+
.slice(0, 19);
1216+
const fraction = (match.groups.fraction || '').padEnd(3, '0');
1217+
return `${utcSeconds}.${fraction}Z`;
1218+
}
1219+
11581220
function isParseableDateOrTimestamp(value: string): boolean {
11591221
return isIsoDate(value) || !Number.isNaN(Date.parse(value));
11601222
}
11611223

1224+
function setPortableExportClockForTests(clock?: () => Date): void {
1225+
portableExportClock = clock || (() => new Date());
1226+
}
1227+
11621228
function validateDateField(
11631229
record: JsonRecord,
11641230
field: string,
@@ -1234,8 +1300,8 @@ async function validatePortableExport(exportDir: string): Promise<ValidationResu
12341300
if (manifest.export_format_version !== EXPORT_FORMAT_VERSION) {
12351301
errors.push(`manifest export_format_version must be ${EXPORT_FORMAT_VERSION}`);
12361302
}
1237-
if (typeof manifest.generated_at !== 'string' || !isParseableDateOrTimestamp(manifest.generated_at)) {
1238-
errors.push('manifest generated_at must be a parseable date or timestamp');
1303+
if (typeof manifest.generated_at !== 'string' || !isRfc3339Instant(manifest.generated_at, true)) {
1304+
errors.push('manifest generated_at must be a UTC RFC3339 date-time');
12391305
}
12401306

12411307
const recordsByEntity: Partial<Record<ExportEntityName, JsonRecord[]>> = {};
@@ -1858,6 +1924,7 @@ export {
18581924
REDACTIONS,
18591925
SCHEMA_VERSION,
18601926
dryRunImport,
1927+
setPortableExportClockForTests,
18611928
validatePortableExport,
18621929
writePortableExport,
18631930
};

‎backend/tests/export-archive.test.ts‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { getClient } from '../src/db/client';
1010
import { startLocal, stopLocal } from '../scripts/local-dynamodb';
1111
import { createTables } from '../scripts/local-dynamodb';
1212
import { createTask } from '../src/db/tasks';
13-
import { validatePortableExport } from '../src/export/portable';
13+
import { setPortableExportClockForTests, validatePortableExport } from '../src/export/portable';
1414
import {
1515
buildArchiveKey,
1616
extractExportArchive,
@@ -99,27 +99,38 @@ describe('offsite portable export archives', () => {
9999
},
100100
};
101101

102-
const result = await writePortableExportArchive(client, {
103-
bucket: 'dataops-v1-export-archives',
104-
prefix: 'exports',
105-
environment: 'prod',
106-
tempDir: path.join(tmpDir, 's3-working-export'),
107-
s3Client: mockS3,
108-
});
109-
110-
assert.strictEqual(result.archiveBucket, 'dataops-v1-export-archives');
111-
assert.match(result.archiveUri, /^s3:\/\/dataops-v1-export-archives\/exports\/prod\//);
112-
assert.ok(result.manifest.redactions.includes('proposal_presentations.action_token_hash'));
113-
assert.ok(result.manifest.omitted_entities.includes('provider_credentials'));
114-
assert.doesNotMatch(result.archiveUri, /[?&](token|credential|signature)=/i);
115-
assert.strictEqual(sentCommands.length, 1);
116-
assert.ok(sentCommands[0] instanceof PutObjectCommand);
117-
const input = (sentCommands[0] as PutObjectCommand).input;
118-
assert.strictEqual(input.Bucket, 'dataops-v1-export-archives');
119-
assert.strictEqual(input.Key, result.archiveKey);
120-
assert.strictEqual(input.ServerSideEncryption, 'AES256');
121-
assert.strictEqual(input.ContentType, 'application/gzip');
122-
assert.ok(input.Body instanceof Buffer);
102+
setPortableExportClockForTests(() => new Date('2026-06-27T12:15:30+02:00'));
103+
try {
104+
const result = await writePortableExportArchive(client, {
105+
bucket: 'dataops-v1-export-archives',
106+
prefix: 'exports',
107+
environment: 'prod',
108+
tempDir: path.join(tmpDir, 's3-working-export'),
109+
s3Client: mockS3,
110+
});
111+
112+
assert.strictEqual(result.archiveBucket, 'dataops-v1-export-archives');
113+
assert.match(result.archiveUri, /^s3:\/\/dataops-v1-export-archives\/exports\/prod\//);
114+
assert.ok(result.manifest.redactions.includes('proposal_presentations.action_token_hash'));
115+
assert.ok(result.manifest.omitted_entities.includes('provider_credentials'));
116+
assert.doesNotMatch(result.archiveUri, /[?&](token|credential|signature)=/i);
117+
assert.strictEqual(sentCommands.length, 1);
118+
assert.ok(sentCommands[0] instanceof PutObjectCommand);
119+
const input = (sentCommands[0] as PutObjectCommand).input;
120+
assert.strictEqual(input.Bucket, 'dataops-v1-export-archives');
121+
assert.strictEqual(input.Key, result.archiveKey);
122+
assert.strictEqual(result.manifest.generated_at, '2026-06-27T10:15:30.000Z');
123+
assert.strictEqual(
124+
input.Key,
125+
'exports/prod/2026-06-27/dataops-execution-2026-06-27T10-15-30-000Z.tar.gz',
126+
);
127+
assert.strictEqual(input.Metadata?.generated_at, result.manifest.generated_at);
128+
assert.strictEqual(input.ServerSideEncryption, 'AES256');
129+
assert.strictEqual(input.ContentType, 'application/gzip');
130+
assert.ok(input.Body instanceof Buffer);
131+
} finally {
132+
setPortableExportClockForTests();
133+
}
123134
});
124135

125136
it('builds deterministic audit-friendly archive keys without private data', () => {

0 commit comments

Comments
 (0)