From 64b17b8024725d35eb71ed912a901ea1a7633af1 Mon Sep 17 00:00:00 2001 From: "Lucas C. Villa Real" Date: Mon, 12 Dec 2022 15:36:34 -0300 Subject: [PATCH 1/4] New sqlite3_module method: xPrepareSql() This commit adds a new method named xPrepareSql to sqlite3_module and an associated opcode VPrepareSql. That opcode is emitted immediately before a VFilter opcode when querying virtual tables. xPrepareSql takes two arguments: a pointer to the virtual table's cursor and, missing from existing sqlite3_module methods, the SQL string being executed by the application. Prior to the introduction of this method, a virtual table that mirrored a remote table had no way to tell which columns of the remote server were requested by the application. Therefore, such an implementation would typically `SELECT *` from the remote table and let callbacks such as xColumn() and xNext() decide which data to send back to the application. The problem with this approach is that queries that SELECT just a subset of the remote columns trigger the transfer of data over the network even though they won't be used. xPrepareSql gives an opportunity for the virtual table implementation to inspect the query string and selectively choose which columns from the remote server to pull and cache locally. Because this change introduces a new method to sqlite3_module, new modules wishing to implement a callback for xPrepareSql must declare a module version (iVersion) 4 or above. --- src/sqlite.h.in | 2 ++ src/test8.c | 17 +++++++++++++++-- src/vdbe.c | 34 ++++++++++++++++++++++++++++++++++ src/wherecode.c | 2 ++ test/rowvaluevtab.test | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7d6813b2af..d7f4598598 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7076,6 +7076,8 @@ struct sqlite3_module { /* The methods above are in versions 1 and 2 of the sqlite_module object. ** Those below are for version 3 and greater. */ int (*xShadowName)(const char*); + /** The methods below are for version 4 and greater. */ + int (*xPrepareSql)(sqlite3_vtab_cursor*, const char*); }; /* diff --git a/src/test8.c b/src/test8.c index 7a532346ed..e7d585d78c 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1293,6 +1293,17 @@ static int echoRollbackTo(sqlite3_vtab *pVTab, int iSavepoint){ return SQLITE_OK; } +static int echoShadowName(const char *name){ + assert( name ); + return SQLITE_OK; +} + +static int echoPrepareSql(sqlite3_vtab_cursor *cur, const char *sql){ + assert( cur ); + assert( sql ); + return SQLITE_OK; +} + /* ** A virtual table module that merely "echos" the contents of another ** table (like an SQL VIEW). @@ -1321,7 +1332,7 @@ static sqlite3_module echoModule = { }; static sqlite3_module echoModuleV2 = { - 2, /* iVersion */ + 4, /* iVersion */ echoCreate, echoConnect, echoBestIndex, @@ -1343,7 +1354,9 @@ static sqlite3_module echoModuleV2 = { echoRename, /* xRename - rename the table */ echoSavepoint, echoRelease, - echoRollbackTo + echoRollbackTo, + echoShadowName, + echoPrepareSql, }; /* diff --git a/src/vdbe.c b/src/vdbe.c index 7f79ef5293..77c18c4c80 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8072,6 +8072,40 @@ case OP_VInitIn: { /* out2 */ } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VPrepareSql P1 * * * * +** +** P1 is a cursor opened using VOpen. +** +** This opcode invokes the xPrepareSql method on the virtual table specified +** by P1. The SQL text parameter to xPrepareSql is obtained from Vdbe* `p`. +** +*/ +case OP_VPrepareSql: { + const sqlite3_module *pModule; + sqlite3_vtab_cursor *pVCur; + sqlite3_vtab *pVtab; + VdbeCursor *pCur; + + pCur = p->apCsr[pOp->p1]; + assert( pCur!=0 ); + assert( pCur->eCurType==CURTYPE_VTAB ); + pVCur = pCur->uc.pVCur; + pVtab = pVCur->pVtab; + pModule = pVtab->pModule; + + /* Invoke the xPrepareSql method */ + if( pModule->iVersion>=4 ){ + if( pModule->xPrepareSql && p->zSql ){ + rc = pModule->xPrepareSql(pVCur, p->zSql); + if( rc ) goto abort_due_to_error; // TODO set and test error msg + } + } + + if( rc ) goto abort_due_to_error; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VFilter P1 P2 P3 P4 * diff --git a/src/wherecode.c b/src/wherecode.c index e36d1c9964..c793c6e33a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1415,6 +1415,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( int addrNotFound; int nConstraint = pLoop->nLTerm; + sqlite3VdbeAddOp1(v, OP_VPrepareSql, iCur); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; for(j=0; j Date: Mon, 19 Dec 2022 11:42:43 -0300 Subject: [PATCH 2/4] vdbeExec: remove redundant goto on error path. This commit also removes a comment that was left over on the previous changeset. --- src/vdbe.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vdbe.c b/src/vdbe.c index 77c18c4c80..9502996eb8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8098,11 +8098,10 @@ case OP_VPrepareSql: { if( pModule->iVersion>=4 ){ if( pModule->xPrepareSql && p->zSql ){ rc = pModule->xPrepareSql(pVCur, p->zSql); - if( rc ) goto abort_due_to_error; // TODO set and test error msg + if( rc ) goto abort_due_to_error; } } - if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ From 9e10fd8db837d051603f60b6dadaa51055f41449 Mon Sep 17 00:00:00 2001 From: "Lucas C. Villa Real" Date: Mon, 19 Dec 2022 12:06:52 -0300 Subject: [PATCH 3/4] xPrepareSql: require iVersion >= 700 Following @psarna's advice, use iVersion >= 700 for community-contributed methods. --- src/sqlite.h.in | 3 ++- src/test8.c | 2 +- src/vdbe.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d7f4598598..0e7fb8d8ff 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7076,7 +7076,8 @@ struct sqlite3_module { /* The methods above are in versions 1 and 2 of the sqlite_module object. ** Those below are for version 3 and greater. */ int (*xShadowName)(const char*); - /** The methods below are for version 4 and greater. */ + /* The methods below relate to features contributed by the community and + ** are available for version 700 and greater. */ int (*xPrepareSql)(sqlite3_vtab_cursor*, const char*); }; diff --git a/src/test8.c b/src/test8.c index e7d585d78c..e5fb869b7c 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1332,7 +1332,7 @@ static sqlite3_module echoModule = { }; static sqlite3_module echoModuleV2 = { - 4, /* iVersion */ + 700, /* iVersion */ echoCreate, echoConnect, echoBestIndex, diff --git a/src/vdbe.c b/src/vdbe.c index 9502996eb8..fa3363d763 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8095,7 +8095,7 @@ case OP_VPrepareSql: { pModule = pVtab->pModule; /* Invoke the xPrepareSql method */ - if( pModule->iVersion>=4 ){ + if( pModule->iVersion>=700 ){ if( pModule->xPrepareSql && p->zSql ){ rc = pModule->xPrepareSql(pVCur, p->zSql); if( rc ) goto abort_due_to_error; From 25e220489c9f6e94a1432f755f3a7c6183100eac Mon Sep 17 00:00:00 2001 From: "Lucas C. Villa Real" Date: Mon, 19 Dec 2022 13:08:40 -0300 Subject: [PATCH 4/4] Rename method from PrepareSql to PreparedSql This name change is motivated to prevent misunderstandings: the callback is not meant to output a prepared SQL statement. Rather, the callback is being informed of the SQL statement prepared by SQLite. --- src/sqlite.h.in | 2 +- src/test8.c | 4 ++-- src/vdbe.c | 14 +++++++------- src/wherecode.c | 2 +- test/rowvaluevtab.test | 12 ++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0e7fb8d8ff..01b605943c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7078,7 +7078,7 @@ struct sqlite3_module { int (*xShadowName)(const char*); /* The methods below relate to features contributed by the community and ** are available for version 700 and greater. */ - int (*xPrepareSql)(sqlite3_vtab_cursor*, const char*); + int (*xPreparedSql)(sqlite3_vtab_cursor*, const char*); }; /* diff --git a/src/test8.c b/src/test8.c index e5fb869b7c..ec6ea3f6a0 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1298,7 +1298,7 @@ static int echoShadowName(const char *name){ return SQLITE_OK; } -static int echoPrepareSql(sqlite3_vtab_cursor *cur, const char *sql){ +static int echoPreparedSql(sqlite3_vtab_cursor *cur, const char *sql){ assert( cur ); assert( sql ); return SQLITE_OK; @@ -1356,7 +1356,7 @@ static sqlite3_module echoModuleV2 = { echoRelease, echoRollbackTo, echoShadowName, - echoPrepareSql, + echoPreparedSql, }; /* diff --git a/src/vdbe.c b/src/vdbe.c index fa3363d763..82e758ef9f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8073,15 +8073,15 @@ case OP_VInitIn: { /* out2 */ #endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VPrepareSql P1 * * * * +/* Opcode: VPreparedSql P1 * * * * ** ** P1 is a cursor opened using VOpen. ** -** This opcode invokes the xPrepareSql method on the virtual table specified -** by P1. The SQL text parameter to xPrepareSql is obtained from Vdbe* `p`. +** This opcode invokes the xPreparedSql method on the virtual table specified +** by P1. The SQL text parameter to xPreparedSql is obtained from Vdbe* `p`. ** */ -case OP_VPrepareSql: { +case OP_VPreparedSql: { const sqlite3_module *pModule; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; @@ -8094,10 +8094,10 @@ case OP_VPrepareSql: { pVtab = pVCur->pVtab; pModule = pVtab->pModule; - /* Invoke the xPrepareSql method */ + /* Invoke the xPreparedSql method */ if( pModule->iVersion>=700 ){ - if( pModule->xPrepareSql && p->zSql ){ - rc = pModule->xPrepareSql(pVCur, p->zSql); + if( pModule->xPreparedSql && p->zSql ){ + rc = pModule->xPreparedSql(pVCur, p->zSql); if( rc ) goto abort_due_to_error; } } diff --git a/src/wherecode.c b/src/wherecode.c index c793c6e33a..21d2cd79b5 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1415,7 +1415,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int addrNotFound; int nConstraint = pLoop->nLTerm; - sqlite3VdbeAddOp1(v, OP_VPrepareSql, iCur); + sqlite3VdbeAddOp1(v, OP_VPreparedSql, iCur); iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; diff --git a/test/rowvaluevtab.test b/test/rowvaluevtab.test index 21a50f8431..862d16e4f9 100644 --- a/test/rowvaluevtab.test +++ b/test/rowvaluevtab.test @@ -96,9 +96,9 @@ do_vfilter4_test 1.4f { SELECT a FROM e1 WHERE (b, c) IN ( VALUES(2, 2) ) } {{SELECT rowid, a, b, c FROM 't1' WHERE b = ?}} -###################################################################### -# Test echo_v2. We simply want to ensure that OP_VPrepareSql executes -###################################################################### +####################################################################### +# Test echo_v2. We simply want to ensure that OP_VPreparedSql executes +####################################################################### do_execsql_test 2.0 { CREATE TABLE t2(a, b, c); @@ -113,10 +113,10 @@ do_execsql_test 2.0 { CREATE VIRTUAL TABLE e2 USING echo_v2(t2); } -proc do_vpreparesql1_test {tn sql expected} { +proc do_vpreparedsql1_test {tn sql expected} { set rc -1 db eval "explain $sql" { - if {$opcode=="VPrepareSql"} { + if {$opcode=="VPreparedSql"} { set rc 0 } } @@ -128,7 +128,7 @@ proc do_vpreparesql1_test {tn sql expected} { do_execsql_test 2.1 { SELECT a FROM e2 WHERE (b, c) IN ( VALUES(1, 3) ) } {three} -do_vpreparesql1_test 2.1f { +do_vpreparedsql1_test 2.1f { SELECT a FROM e2 WHERE (b, c) IN ( VALUES(2, 2) ) } {0}