diff --git a/.changeset/sync-rules-ascii-upper-lower.md b/.changeset/sync-rules-ascii-upper-lower.md new file mode 100644 index 000000000..500e99a58 --- /dev/null +++ b/.changeset/sync-rules-ascii-upper-lower.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-sync-rules': patch +--- + +Make `upper()` and `lower()` ASCII-only in the JS evaluator to match SQLite's default behaviour. Previously the server used `String.prototype.toUpperCase()` / `.toLowerCase()`, which are Unicode-aware and perform length-changing case folds (ß -> SS, fi -> FI, İ -> İ̇). The client SQLite uses ASCII-only semantics for the same calls, so server-side bucket keys silently disagreed with client-side parameter values for any source data containing non-ASCII letters. diff --git a/packages/sync-rules/src/sql_functions.ts b/packages/sync-rules/src/sql_functions.ts index 26382feb3..ca6cb0c4f 100644 --- a/packages/sync-rules/src/sql_functions.ts +++ b/packages/sync-rules/src/sql_functions.ts @@ -60,30 +60,62 @@ export function getOperatorFunction(op: string): SqlFunction { }; } +/** + * SQLite's default `upper()` and `lower()` are ASCII-only: only `a-z` <-> `A-Z` + * are converted; all other characters (including length-changing case folds + * like ß -> SS, fi -> FI, I-dot) pass through unchanged. + * + * `String.prototype.toUpperCase()` / `.toLowerCase()` in JavaScript are + * Unicode-aware and DO perform length-changing folds, so an `upper()` or + * `lower()` call evaluated server-side here produces a different string than + * the same call run client-side against SQLite. Bucket keys silently desync; + * rows containing non-ASCII letters end up routed to the wrong bucket. + * + * Same class of bug as the merged Sync Streams correctness fixes + * #644 / #645 / #646 / #647. + */ +function asciiToUpper(text: string): string { + let out = ''; + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + out += code >= 97 && code <= 122 ? String.fromCharCode(code - 32) : text[i]; + } + return out; +} + +function asciiToLower(text: string): string { + let out = ''; + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + out += code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : text[i]; + } + return out; +} + const upper: DocumentedSqlFunction = { debugName: 'upper', call(value: SqliteValue) { const text = castAsText(value); - return text?.toUpperCase() ?? null; + return text == null ? null : asciiToUpper(text); }, parameters: [{ name: 'value', type: ExpressionType.ANY, optional: false }], getReturnType(args) { return ExpressionType.TEXT; }, - detail: 'Convert text to upper case' + detail: 'Convert ASCII a-z to A-Z (matches SQLite default; non-ASCII passes through)' }; const lower: DocumentedSqlFunction = { debugName: 'lower', call(value: SqliteValue) { const text = castAsText(value); - return text?.toLowerCase() ?? null; + return text == null ? null : asciiToLower(text); }, parameters: [{ name: 'value', type: ExpressionType.ANY, optional: false }], getReturnType(args) { return ExpressionType.TEXT; }, - detail: 'Convert text to lower case' + detail: 'Convert ASCII A-Z to a-z (matches SQLite default; non-ASCII passes through)' }; const substring: DocumentedSqlFunction = { diff --git a/packages/sync-rules/test/src/sync_plan/evaluator/sqlite_semantics.test.ts b/packages/sync-rules/test/src/sync_plan/evaluator/sqlite_semantics.test.ts index 209468f51..2cf5decb7 100644 --- a/packages/sync-rules/test/src/sync_plan/evaluator/sqlite_semantics.test.ts +++ b/packages/sync-rules/test/src/sync_plan/evaluator/sqlite_semantics.test.ts @@ -4,6 +4,42 @@ import { requestParameters, TestSourceTable } from '../../util.js'; import { syncTest } from './utils.js'; describe('operators match SQLite', () => { + syncTest('upper / lower use ASCII-only semantics (matches SQLite default)', ({ sync }) => { + // SQLite's default upper()/lower() only handles a-z / A-Z; non-ASCII + // letters pass through unchanged. JavaScript's toUpperCase/toLowerCase + // are Unicode-aware and length-changing (ß -> SS, fi -> FI). When the + // evaluator and the client disagree on the result of upper(), bucket + // keys silently diverge and rows are routed to the wrong bucket. + const streams = sync.prepareSyncStreams(` +config: + edition: 3 + +streams: + a: + query: 'SELECT id, UPPER(name) AS upper, LOWER(name) AS lower FROM tbl' +`); + + const table = new TestSourceTable('tbl'); + + function evaluate(name: SqliteValue) { + const [row] = streams.evaluateRow({ sourceTable: table, record: { id: 'ignored', name } }); + return { upper: row.data['upper'], lower: row.data['lower'] }; + } + + // ASCII works exactly as before. + expect(evaluate('hello')).toStrictEqual({ upper: 'HELLO', lower: 'hello' }); + expect(evaluate('Hello World')).toStrictEqual({ upper: 'HELLO WORLD', lower: 'hello world' }); + + // Non-ASCII letters now pass through unchanged (matching SQLite), + // instead of being length-changed by JS Unicode folding. + expect(evaluate('straße')).toStrictEqual({ upper: 'STRAßE', lower: 'straße' }); + expect(evaluate('file')).toStrictEqual({ upper: 'fiLE', lower: 'file' }); + + // Length is preserved (was previously length-changed by JS folds). + expect(evaluate('straße').upper).toHaveLength('straße'.length); + expect(evaluate('file').upper).toHaveLength('file'.length); + }); + syncTest('division by zero', ({ sync }) => { // Regression test for https://github.com/powersync-ja/powersync-service/pull/646. const streams = sync.prepareSyncStreams(`