From 25534480311718da074c8f92cea20c3efa16dd92 Mon Sep 17 00:00:00 2001 From: pandor4u <103976470+pandor4u@users.noreply.github.com> Date: Fri, 10 Apr 2026 17:46:36 -0500 Subject: [PATCH 1/3] feat: PostgreSQL-backed search/document storage as ElasticSearch alternative Simplified PR addressing review feedback: - Consolidated duplicated INSERT...ON CONFLICT SQL into shared constant - Removed fork-specific files (AGENTS.md, CLAUDE.md, GEMINI.md, docker compose, .gitignore) - Moved PostgreSQL tests to separate opt-in suite (MoquiSuite untouched) - Updated ElasticRequestLogFilter to use ElasticClient interface type - 83 tests: 46 unit (query translation + SQL injection) + 37 integration New files: - PostgresElasticClient.groovy: Full ElasticClient impl using JSONB/tsvector - ElasticQueryTranslator.groovy: ES Query DSL to PostgreSQL SQL - PostgresSearchLogger.groovy: Log4j2 appender for PostgreSQL - SearchEntities.xml: Entity definitions for search tables - PostgresSearchSuite.groovy: Separate JUnit test suite - PostgresSearchTranslatorTests.groovy: Unit tests - PostgresElasticClientTests.groovy: Integration tests Modified files: - ElasticFacadeImpl.groovy: type=postgres routing - ElasticRequestLogFilter.groovy: Interface type usage - MoquiDefaultConf.xml: Postgres config + entity load - moqui-conf-3.xsd: type attribute with elastic/postgres enum - build.gradle: Test dependencies and suite include --- framework/build.gradle | 3 + framework/entity/SearchEntities.xml | 115 ++ .../impl/context/ElasticFacadeImpl.groovy | 54 +- .../context/ElasticQueryTranslator.groovy | 675 ++++++++++ .../impl/context/PostgresElasticClient.groovy | 1176 +++++++++++++++++ .../impl/util/PostgresSearchLogger.groovy | 244 ++++ .../webapp/ElasticRequestLogFilter.groovy | 10 +- .../src/main/resources/MoquiDefaultConf.xml | 11 +- .../groovy/PostgresElasticClientTests.groovy | 701 ++++++++++ .../test/groovy/PostgresSearchSuite.groovy | 30 + .../PostgresSearchTranslatorTests.groovy | 478 +++++++ framework/xsd/moqui-conf-3.xsd | 25 +- 12 files changed, 3491 insertions(+), 31 deletions(-) create mode 100644 framework/entity/SearchEntities.xml create mode 100644 framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy create mode 100644 framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy create mode 100644 framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy create mode 100644 framework/src/test/groovy/PostgresElasticClientTests.groovy create mode 100644 framework/src/test/groovy/PostgresSearchSuite.groovy create mode 100644 framework/src/test/groovy/PostgresSearchTranslatorTests.groovy diff --git a/framework/build.gradle b/framework/build.gradle index 335fe6a97..d4d00afd9 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -171,6 +171,8 @@ dependencies { testImplementation 'org.junit.platform:junit-platform-suite:6.0.1' // junit-jupiter-api for using JUnit directly, not generally needed for Spock based tests testImplementation 'org.junit.jupiter:junit-jupiter-api:6.0.1' + // junit-jupiter-engine required to execute @Test-annotated methods via JUnit Platform + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:6.0.1' // Spock Framework testImplementation platform('org.spockframework:spock-bom:2.4-groovy-5.0') // Apache 2.0 testImplementation 'org.spockframework:spock-core:2.4-groovy-5.0' // Apache 2.0 @@ -201,6 +203,7 @@ test { dependsOn cleanTest include '**/*MoquiSuite.class' + include '**/*PostgresSearchSuite.class' systemProperty 'moqui.runtime', '../runtime' systemProperty 'moqui.conf', 'conf/MoquiDevConf.xml' diff --git a/framework/entity/SearchEntities.xml b/framework/entity/SearchEntities.xml new file mode 100644 index 000000000..0ed222b5c --- /dev/null +++ b/framework/entity/SearchEntities.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy index 72ffd6ddc..8f26ba848 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy @@ -32,6 +32,7 @@ import org.moqui.impl.entity.EntityDefinition import org.moqui.impl.entity.EntityJavaUtil import org.moqui.impl.entity.FieldInfo import org.moqui.impl.util.ElasticSearchLogger +import org.moqui.impl.util.PostgresSearchLogger import org.moqui.util.LiteStringMap import org.moqui.util.MNode import org.moqui.util.RestClient @@ -69,8 +70,9 @@ class ElasticFacadeImpl implements ElasticFacade { } public final ExecutionContextFactoryImpl ecfi - private final Map clientByClusterName = new LinkedHashMap<>() + private final Map clientByClusterName = new LinkedHashMap<>() private ElasticSearchLogger esLogger = null + private PostgresSearchLogger pgLogger = null ElasticFacadeImpl(ExecutionContextFactoryImpl ecfi) { this.ecfi = ecfi @@ -90,14 +92,21 @@ class ElasticFacadeImpl implements ElasticFacade { logger.warn("ElasticFacade Client for cluster ${clusterName} already initialized, skipping") continue } - if (!clusterUrl) { - logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping") - continue - } + String clusterType = clusterNode.attribute("type") ?: "elastic" try { - ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi) - clientByClusterName.put(clusterName, elci) + if ("postgres".equals(clusterType)) { + PostgresElasticClient pgc = new PostgresElasticClient(clusterNode, ecfi) + clientByClusterName.put(clusterName, pgc) + logger.info("Initialized PostgresElasticClient for cluster ${clusterName}") + } else { + if (!clusterUrl) { + logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping") + continue + } + ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi) + clientByClusterName.put(clusterName, elci) + } } catch (Throwable t) { Throwable cause = t.getCause() if (cause != null && cause.message.contains("refused")) { @@ -108,22 +117,29 @@ class ElasticFacadeImpl implements ElasticFacade { } } - // init ElasticSearchLogger - if (esLogger == null || !esLogger.isInitialized()) { - ElasticClientImpl loggerEci = clientByClusterName.get("logger") ?: clientByClusterName.get("default") - if (loggerEci != null) { - logger.info("Initializing ElasticSearchLogger with cluster ${loggerEci.getClusterName()}") - esLogger = new ElasticSearchLogger(loggerEci, ecfi) + // init ElasticSearchLogger / PostgresSearchLogger depending on backend type + ElasticClient loggerClient = clientByClusterName.get("logger") ?: clientByClusterName.get("default") + if (loggerClient instanceof PostgresElasticClient) { + if (pgLogger == null || !pgLogger.isInitialized()) { + logger.info("Initializing PostgresSearchLogger with cluster ${loggerClient.getClusterName()}") + pgLogger = new PostgresSearchLogger((PostgresElasticClient) loggerClient, ecfi) + } else { + logger.warn("PostgresSearchLogger in place and initialized, skipping") + } + } else if (loggerClient instanceof ElasticClientImpl) { + if (esLogger == null || !esLogger.isInitialized()) { + logger.info("Initializing ElasticSearchLogger with cluster ${loggerClient.getClusterName()}") + esLogger = new ElasticSearchLogger((ElasticClientImpl) loggerClient, ecfi) } else { - logger.warn("No Elastic Client found with name 'logger' or 'default', not initializing ElasticSearchLogger") + logger.warn("ElasticSearchLogger in place and initialized, skipping") } } else { - logger.warn("ElasticSearchLogger in place and initialized, not initializing ElasticSearchLogger") + logger.warn("No Elastic/Postgres Client found with name 'logger' or 'default', not initializing search logger") } // Index DataFeed with indexOnStartEmpty=Y try { - ElasticClientImpl defaultEci = clientByClusterName.get("default") + ElasticClient defaultEci = clientByClusterName.get("default") if (defaultEci != null) { EntityList dataFeedList = ecfi.entityFacade.find("moqui.entity.feed.DataFeed") .condition("indexOnStartEmpty", "Y").disableAuthz().list() @@ -151,7 +167,11 @@ class ElasticFacadeImpl implements ElasticFacade { void destroy() { if (esLogger != null) esLogger.destroy() - for (ElasticClientImpl eci in clientByClusterName.values()) eci.destroy() + if (pgLogger != null) pgLogger.destroy() + for (ElasticClient eci in clientByClusterName.values()) { + if (eci instanceof ElasticClientImpl) ((ElasticClientImpl) eci).destroy() + else if (eci instanceof PostgresElasticClient) ((PostgresElasticClient) eci).destroy() + } } @Override ElasticClient getDefault() { return clientByClusterName.get("default") } diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy new file mode 100644 index 000000000..887aa1237 --- /dev/null +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy @@ -0,0 +1,675 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.impl.context + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * Translates ElasticSearch/OpenSearch Query DSL (Map structures) into PostgreSQL SQL WHERE clauses, + * ORDER BY expressions, and OFFSET/LIMIT pagination for use by PostgresElasticClient. + * + * Supports the query types used by Moqui's SearchServices.xml and entity condition makeSearchFilter() methods: + * - query_string (→ websearch_to_tsquery / plainto_tsquery on content_tsv) + * - bool (must / should / must_not / filter) + * - term, terms + * - range + * - match_all + * - exists + * - nested (→ jsonb_array_elements EXISTS subquery) + */ +class ElasticQueryTranslator { + private final static Logger logger = LoggerFactory.getLogger(ElasticQueryTranslator.class) + + /** Regex pattern for valid field names — alphanumeric, underscores, dots, hyphens, and @ (for @timestamp) */ + private static final java.util.regex.Pattern SAFE_FIELD_PATTERN = java.util.regex.Pattern.compile('^[a-zA-Z0-9_@][a-zA-Z0-9_.\\-]*$') + + /** + * Validate that a field name is safe for interpolation into SQL. + * Rejects any field containing SQL metacharacters (quotes, semicolons, parentheses, etc.) + * @throws IllegalArgumentException if the field name contains unsafe characters + */ + static String sanitizeFieldName(String field) { + if (field == null || field.isEmpty()) throw new IllegalArgumentException("Field name must not be empty") + if (!SAFE_FIELD_PATTERN.matcher(field).matches()) { + throw new IllegalArgumentException("Unsafe field name rejected: '${field}' — only alphanumeric, underscore, dot, hyphen, and @ allowed") + } + if (field.contains("--")) { + throw new IllegalArgumentException("Unsafe field name rejected: '${field}' — double-hyphen (SQL comment) not allowed") + } + if (field.length() > 256) { + throw new IllegalArgumentException("Field name too long (max 256 chars): '${field}'") + } + return field + } + + /** Holds the result of translating a query DSL fragment or full search request */ + static class TranslatedQuery { + /** SQL WHERE clause fragment (without the "WHERE" keyword), or "TRUE" if no filter */ + String whereClause = "TRUE" + /** JDBC bind parameters in order corresponding to ? placeholders in whereClause */ + List params = [] + /** SQL ORDER BY expression (without the "ORDER BY" keyword), or null */ + String orderBy = null + /** The tsquery expression (as SQL expression string) for use in ts_rank_cd() and ts_headline() */ + String tsqueryExpr = null + /** Bind parameters specifically for tsqueryExpr (separate from WHERE params) */ + List tsqueryParams = [] + /** OFFSET value for pagination */ + int fromOffset = 0 + /** LIMIT value for pagination */ + int sizeLimit = 20 + /** Track total hits (adds no SQL change but reflects ES track_total_hits flag) */ + boolean trackTotal = true + /** Fields to highlight, keyed by field name */ + Map highlightFields = [:] + } + + /** + * Translate a full ES searchMap (the body sent to /_search) into a TranslatedQuery. + * @param searchMap Map as built by SearchServices.search#DataDocuments + */ + static TranslatedQuery translateSearchMap(Map searchMap) { + TranslatedQuery tq = new TranslatedQuery() + + // Pagination + Object fromVal = searchMap.get("from") + if (fromVal != null) tq.fromOffset = ((Number) fromVal).intValue() + Object sizeVal = searchMap.get("size") + if (sizeVal != null) tq.sizeLimit = ((Number) sizeVal).intValue() + + // Sort + Object sortVal = searchMap.get("sort") + if (sortVal instanceof List) { + tq.orderBy = translateSort((List) sortVal) + } + + // Highlight fields + Object highlightVal = searchMap.get("highlight") + if (highlightVal instanceof Map) { + Object fieldsVal = ((Map) highlightVal).get("fields") + if (fieldsVal instanceof Map) tq.highlightFields = (Map) fieldsVal + } + + // track_total_hits + Object tthVal = searchMap.get("track_total_hits") + if (tthVal != null) tq.trackTotal = Boolean.TRUE == tthVal || "true".equals(tthVal.toString()) + + // Query + Object queryVal = searchMap.get("query") + if (queryVal instanceof Map) { + QueryResult qr = translateQuery((Map) queryVal) + tq.whereClause = qr.clause ?: "TRUE" + tq.params = qr.params + tq.tsqueryExpr = qr.tsqueryExpr + tq.tsqueryParams = qr.tsqueryParams + } + + return tq + } + + /** Internal result holder for a single query fragment */ + static class QueryResult { + String clause = "TRUE" + List params = [] + /** If this query has a full-text component, the SQL tsquery expression for scoring/highlighting */ + String tsqueryExpr = null + /** Bind parameters specifically for tsqueryExpr (separate from WHERE clause params) */ + List tsqueryParams = [] + } + + static QueryResult translateQuery(Map queryMap) { + if (queryMap == null || queryMap.isEmpty()) return new QueryResult() + + String queryType = (String) queryMap.keySet().iterator().next() + Object queryVal = queryMap.get(queryType) + + switch (queryType) { + case "match_all": return translateMatchAll() + case "match_none": + QueryResult qr = new QueryResult(); qr.clause = "FALSE"; return qr + case "query_string": return translateQueryString((Map) queryVal) + case "multi_match": return translateMultiMatch((Map) queryVal) + case "bool": return translateBool((Map) queryVal) + case "term": return translateTerm((Map) queryVal, false) + case "terms": return translateTerms((Map) queryVal) + case "range": return translateRange((Map) queryVal) + case "exists": return translateExists((Map) queryVal) + case "nested": return translateNested((Map) queryVal) + case "ids": return translateIds((Map) queryVal) + default: + logger.warn("ElasticQueryTranslator: unsupported query type '${queryType}', using TRUE") + return new QueryResult() + } + } + + private static QueryResult translateMatchAll() { + QueryResult qr = new QueryResult() + qr.clause = "TRUE" + return qr + } + + private static QueryResult translateQueryString(Map qsMap) { + QueryResult qr = new QueryResult() + if (qsMap == null) return qr + + String query = (String) qsMap.get("query") + if (!query || query.trim().isEmpty()) return qr + + // Clean up the query string: + // 1. Lucene field:value syntax → handle field-specific searches + // 2. Strip unsupported operators, translate AND/OR/NOT + // 3. Use websearch_to_tsquery which supports quoted phrases, AND, OR, -, + + String cleanedQuery = cleanLuceneQuery(query) + + if (!cleanedQuery || cleanedQuery.trim().isEmpty()) return qr + + // Use websearch_to_tsquery for natural language queries + // It handles: "exact phrase", AND/OR/NOT, +required, -exclude + qr.tsqueryExpr = "websearch_to_tsquery('english', ?)" + qr.tsqueryParams = [cleanedQuery] + qr.params = [cleanedQuery] + qr.clause = "content_tsv @@ websearch_to_tsquery('english', ?)" + return qr + } + + private static QueryResult translateMultiMatch(Map mmMap) { + // Treat like query_string on all fields + String query = (String) mmMap.get("query") + if (!query) return new QueryResult() + return translateQueryString([query: query]) + } + + private static QueryResult translateBool(Map boolMap) { + QueryResult qr = new QueryResult() + if (boolMap == null) return qr + + List clauses = [] + List params = [] + String combinedTsquery = null + List combinedTsqueryParams = [] + + // must (AND) + Object mustVal = boolMap.get("must") + if (mustVal instanceof List) { + List mustClauses = [] + for (Object item in (List) mustVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + mustClauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = combinedTsquery ? "(${combinedTsquery}) && (${itemQr.tsqueryExpr})" : itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + } + if (mustClauses) clauses.add("(" + mustClauses.join(" AND ") + ")") + } else if (mustVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) mustVal) + clauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + + // filter (same as must for our purposes) + Object filterVal = boolMap.get("filter") + if (filterVal instanceof List) { + List filterClauses = [] + for (Object item in (List) filterVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + filterClauses.add(itemQr.clause) + params.addAll(itemQr.params) + } + } + if (filterClauses) clauses.add("(" + filterClauses.join(" AND ") + ")") + } else if (filterVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) filterVal) + clauses.add(itemQr.clause) + params.addAll(itemQr.params) + } + + // should (OR) + Object shouldVal = boolMap.get("should") + if (shouldVal instanceof List) { + List shouldClauses = [] + for (Object item in (List) shouldVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + shouldClauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = combinedTsquery ? "(${combinedTsquery}) || (${itemQr.tsqueryExpr})" : itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + } + if (shouldClauses) { + int minShouldMatch = 1 + Object msmVal = boolMap.get("minimum_should_match") + if (msmVal != null) minShouldMatch = ((Number) msmVal).intValue() + if (minShouldMatch == 1) { + clauses.add("(" + shouldClauses.join(" OR ") + ")") + } else { + // For minimum_should_match > 1, use a CASE/SUM trick for simplicity just add as OR + clauses.add("(" + shouldClauses.join(" OR ") + ")") + } + } + } else if (shouldVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) shouldVal) + clauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + + // must_not (NOT) + Object mustNotVal = boolMap.get("must_not") + if (mustNotVal instanceof List) { + List mustNotClauses = [] + for (Object item in (List) mustNotVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + mustNotClauses.add(itemQr.clause) + params.addAll(itemQr.params) + } + } + if (mustNotClauses) clauses.add("NOT (" + mustNotClauses.join(" OR ") + ")") + } else if (mustNotVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) mustNotVal) + clauses.add("NOT (${itemQr.clause})") + params.addAll(itemQr.params) + } + + qr.clause = clauses ? "(" + clauses.join(" AND ") + ")" : "TRUE" + qr.params = params + qr.tsqueryExpr = combinedTsquery + qr.tsqueryParams = combinedTsqueryParams + return qr + } + + private static QueryResult translateTerm(Map termMap, boolean ignoreCase) { + QueryResult qr = new QueryResult() + if (termMap == null || termMap.isEmpty()) return qr + + String field = (String) termMap.keySet().iterator().next() + Object valueHolder = termMap.get(field) + Object value + if (valueHolder instanceof Map) { + value = ((Map) valueHolder).get("value") + } else { + value = valueHolder + } + if (value == null) { qr.clause = "TRUE"; return qr } + + // _id is a special ES field that maps to the doc_id column + if (field == "_id") { + qr.clause = "doc_id = ?" + qr.params = [value.toString()] + return qr + } + + String jsonPath = fieldToJsonPath("document", field) + if (ignoreCase && value instanceof String) { + qr.clause = "LOWER(${jsonPath}) = LOWER(?)" + } else { + qr.clause = "${jsonPath} = ?" + } + qr.params = [value.toString()] + return qr + } + + private static QueryResult translateTerms(Map termsMap) { + QueryResult qr = new QueryResult() + if (termsMap == null || termsMap.isEmpty()) return qr + + // Remove boost key if present + Map filteredMap = termsMap.findAll { k, v -> k != "boost" } + if (filteredMap.isEmpty()) return qr + + String field = (String) filteredMap.keySet().iterator().next() + Object valuesObj = filteredMap.get(field) + if (!(valuesObj instanceof List)) { qr.clause = "TRUE"; return qr } + List values = (List) valuesObj + if (values.isEmpty()) { qr.clause = "FALSE"; return qr } + + String jsonPath = fieldToJsonPath("document", field) + List placeholders = values.collect { "?" } + qr.clause = "${jsonPath} IN (${placeholders.join(', ')})" + qr.params = values.collect { it?.toString() } + return qr + } + + private static QueryResult translateRange(Map rangeMap) { + QueryResult qr = new QueryResult() + if (rangeMap == null || rangeMap.isEmpty()) return qr + + String field = (String) rangeMap.keySet().iterator().next() + Object rangeSpec = rangeMap.get(field) + if (!(rangeSpec instanceof Map)) return qr + + Map rangeSpecMap = (Map) rangeSpec + String jsonPath = fieldToJsonPath("document", field) + List conditions = [] + List params = [] + + // Determine cast type based on common field name patterns + String castType = guessCastType(field) + + Object gte = rangeSpecMap.get("gte") + Object gt = rangeSpecMap.get("gt") + Object lte = rangeSpecMap.get("lte") + Object lt = rangeSpecMap.get("lt") + + if (gte != null) { conditions.add("(${jsonPath})${castType} >= ?"); params.add(gte.toString()) } + if (gt != null) { conditions.add("(${jsonPath})${castType} > ?"); params.add(gt.toString()) } + if (lte != null) { conditions.add("(${jsonPath})${castType} <= ?"); params.add(lte.toString()) } + if (lt != null) { conditions.add("(${jsonPath})${castType} < ?"); params.add(lt.toString()) } + + if (conditions.isEmpty()) { qr.clause = "TRUE"; return qr } + qr.clause = conditions.join(" AND ") + qr.params = params + return qr + } + + private static QueryResult translateExists(Map existsMap) { + QueryResult qr = new QueryResult() + if (existsMap == null) return qr + String field = (String) existsMap.get("field") + if (!field) return qr + + // Validate field name to prevent SQL injection + sanitizeFieldName(field) + // For nested paths, check the nested path exists + if (field.contains(".")) { + List parts = field.split("\\.") as List + String topLevel = parts[0] + qr.clause = "document ? '${topLevel}'" + } else { + qr.clause = "document ? '${field}'" + } + return qr + } + + private static QueryResult translateNested(Map nestedMap) { + QueryResult qr = new QueryResult() + if (nestedMap == null) return qr + + String path = (String) nestedMap.get("path") + Map innerQuery = (Map) nestedMap.get("query") + if (!path || !innerQuery) return qr + + // Validate path to prevent SQL injection + sanitizeFieldName(path) + // Translate the inner query against jsonb_array_elements alias "elem" + QueryResult innerQr = translateNestedQuery(innerQuery, path) + qr.clause = "EXISTS (SELECT 1 FROM jsonb_array_elements(document->'${path}') AS elem WHERE ${innerQr.clause})" + qr.params = innerQr.params + return qr + } + + /** Translate a query in the context of a nested jsonb_array_elements expression (uses "elem" alias) */ + private static QueryResult translateNestedQuery(Map queryMap, String parentPath) { + QueryResult qr = new QueryResult() + if (queryMap == null || queryMap.isEmpty()) return qr + + String queryType = (String) queryMap.keySet().iterator().next() + Object queryVal = queryMap.get(queryType) + + if (queryType == "bool") { + return translateNestedBool((Map) queryVal, parentPath) + } else if (queryType == "term") { + return translateNestedTerm((Map) queryVal, parentPath) + } else if (queryType == "terms") { + return translateNestedTerms((Map) queryVal, parentPath) + } else if (queryType == "range") { + return translateNestedRange((Map) queryVal, parentPath) + } else if (queryType == "match_all") { + return new QueryResult() + } else { + logger.warn("ElasticQueryTranslator.translateNestedQuery: unsupported nested query type '${queryType}', using TRUE") + return new QueryResult() + } + } + + private static QueryResult translateNestedBool(Map boolMap, String parentPath) { + QueryResult qr = new QueryResult() + if (boolMap == null) return qr + List clauses = [] + List params = [] + + for (String key in ["must", "filter", "should", "must_not"]) { + Object val = boolMap.get(key) + List items + if (val instanceof List) items = (List) val + else if (val instanceof Map) items = [(Map) val] + else continue + + List itemClauses = [] + for (Map item in items) { + QueryResult ir = translateNestedQuery(item, parentPath) + itemClauses.add(ir.clause) + params.addAll(ir.params) + } + if (itemClauses) { + String joined = "(" + itemClauses.join(" AND ") + ")" + if (key == "must_not") joined = "NOT " + joined + else if (key == "should") joined = "(" + itemClauses.join(" OR ") + ")" + clauses.add(joined) + } + } + qr.clause = clauses ? clauses.join(" AND ") : "TRUE" + qr.params = params + return qr + } + + private static QueryResult translateNestedTerm(Map termMap, String parentPath) { + QueryResult qr = new QueryResult() + if (termMap == null || termMap.isEmpty()) return qr + String field = (String) termMap.keySet().iterator().next() + Object valueHolder = termMap.get(field) + Object value = valueHolder instanceof Map ? ((Map) valueHolder).get("value") : valueHolder + if (value == null) { qr.clause = "TRUE"; return qr } + + // For nested terms "parentPath.field", strip the parent path prefix + String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field + sanitizeFieldName(localField) + qr.clause = "elem->>'${localField}' = ?" + qr.params = [value.toString()] + return qr + } + + private static QueryResult translateNestedTerms(Map termsMap, String parentPath) { + QueryResult qr = new QueryResult() + Map filteredMap = termsMap.findAll { k, v -> k != "boost" } + if (filteredMap.isEmpty()) return qr + String field = (String) filteredMap.keySet().iterator().next() + Object valuesObj = filteredMap.get(field) + if (!(valuesObj instanceof List)) { qr.clause = "TRUE"; return qr } + List values = (List) valuesObj + if (values.isEmpty()) { qr.clause = "FALSE"; return qr } + String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field + sanitizeFieldName(localField) + qr.clause = "elem->>'${localField}' IN (${values.collect { '?' }.join(', ')})" + qr.params = values.collect { it?.toString() } + return qr + } + + private static QueryResult translateNestedRange(Map rangeMap, String parentPath) { + QueryResult qr = new QueryResult() + if (rangeMap == null || rangeMap.isEmpty()) return qr + String field = (String) rangeMap.keySet().iterator().next() + Object rangeSpec = rangeMap.get(field) + if (!(rangeSpec instanceof Map)) return qr + Map rangeSpecMap = (Map) rangeSpec + String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field + sanitizeFieldName(localField) + String castType = guessCastType(localField) + List conditions = [] + List params = [] + Object gte = rangeSpecMap.get("gte"); if (gte != null) { conditions.add("(elem->>'${localField}')${castType} >= ?"); params.add(gte.toString()) } + Object gt = rangeSpecMap.get("gt"); if (gt != null) { conditions.add("(elem->>'${localField}')${castType} > ?"); params.add(gt.toString()) } + Object lte = rangeSpecMap.get("lte"); if (lte != null) { conditions.add("(elem->>'${localField}')${castType} <= ?"); params.add(lte.toString()) } + Object lt = rangeSpecMap.get("lt"); if (lt != null) { conditions.add("(elem->>'${localField}')${castType} < ?"); params.add(lt.toString()) } + qr.clause = conditions ? conditions.join(" AND ") : "TRUE" + qr.params = params + return qr + } + + private static QueryResult translateIds(Map idsMap) { + QueryResult qr = new QueryResult() + Object vals = idsMap?.get("values") + if (!(vals instanceof List) || ((List) vals).isEmpty()) { qr.clause = "FALSE"; return qr } + List ids = (List) vals + qr.clause = "doc_id IN (${ids.collect { '?' }.join(', ')})" + qr.params = ids.collect { it?.toString() } + return qr + } + + /** Translate an ES sort spec (list of sort entries) to a SQL ORDER BY expression */ + static String translateSort(List sortList) { + if (!sortList) return null + List parts = [] + for (Object sortEntry in sortList) { + if (sortEntry instanceof Map) { + Map sortMap = (Map) sortEntry + for (Map.Entry entry in sortMap.entrySet()) { + String field = ((String) entry.key).replace(".keyword", "") + String dir = "ASC" + if (entry.value instanceof Map) { + String orderVal = (String) ((Map) entry.value).get("order") + if ("desc".equalsIgnoreCase(orderVal)) dir = "DESC" + } else if ("desc".equalsIgnoreCase(entry.value?.toString())) { + dir = "DESC" + } + + if ("_score".equals(field)) { + parts.add("_score ${dir}") + } else { + String castType = guessCastType(field) + if (castType) { + parts.add("(${fieldToJsonPath("document", field)})${castType} ${dir}") + } else { + parts.add("${fieldToJsonPath("document", field)} ${dir}") + } + } + } + } else if (sortEntry instanceof String) { + String field = ((String) sortEntry).replace(".keyword", "") + if ("_score".equals(field)) { + parts.add("_score DESC") + } else { + parts.add("${fieldToJsonPath("document", field)} ASC") + } + } + } + return parts ? parts.join(", ") : null + } + + /** + * Convert an ES field path to a PostgreSQL JSONB access expression. + * E.g. "product.name" → "document->'product'->>'name'" + * "productId" → "document->>'productId'" + */ + static String fieldToJsonPath(String docAlias, String field) { + // Strip .keyword suffix (used in ES for exact/sortable text fields) + if (field.endsWith(".keyword")) field = field.substring(0, field.length() - ".keyword".length()) + // Validate field name to prevent SQL injection + sanitizeFieldName(field) + List parts = field.split("\\.") as List + if (parts.size() == 1) return "${docAlias}->>'${field}'" + // For nested paths: docAlias->'part1'->'part2'->>'lastPart' + StringBuilder sb = new StringBuilder(docAlias) + for (int i = 0; i < parts.size() - 1; i++) { + sb.append("->'${parts[i]}'") + } + sb.append("->>'${parts[parts.size() - 1]}'") + return sb.toString() + } + + /** + * Guess the appropriate PostgreSQL cast type for a field name to use in range/sort comparisons. + * Returns empty string if no cast is needed (use text comparison). + */ + private static String guessCastType(String field) { + String lf = field.toLowerCase() + if (lf.contains("date") || lf.contains("stamp") || lf.contains("time") || lf == "@timestamp") { + return "::timestamptz" + } + if (lf.contains("amount") || lf.contains("price") || lf.contains("cost") || lf.contains("total") || + lf.contains("quantity") || lf.contains("qty") || lf.contains("score") || lf.contains("count") || + lf.contains("number") || lf.contains("num") || lf.contains("id") && lf.endsWith("num")) { + return "::numeric" + } + return "" + } + + /** + * Clean up a Lucene query string to be safe for use with websearch_to_tsquery. + * websearch_to_tsquery supports: "quoted phrases", AND, OR, -, + + * This removes/translates Lucene-specific syntax that websearch_to_tsquery doesn't support: + * - field:value (extract field-specific as general text) + * - field:[range TO range] (drop or convert) + * - wildcard * ? (drop trailing wildcards, keep term) + * - boost ^ (strip) + * - fuzzy ~ (strip) + * - parentheses → use natural AND grouping + */ + static String cleanLuceneQuery(String query) { + if (!query) return query + String q = query.trim() + + // Remove Lucene field:value prefixes (keep just the value part) + q = q.replaceAll(/\w+:("(?:[^"\\]|\\.)*"|\S+)/, '$1') + + // Remove range queries [X TO Y] + q = q.replaceAll(/\[[^\]]*\]/, '') + q = q.replaceAll(/\{[^}]*\}/, '') + + // Remove boost operators (^number) + q = q.replaceAll(/\^[\d.]+/, '') + + // Remove fuzzy operators (~number or just ~) + q = q.replaceAll(/~[\d.]*/, '') + + // Normalize AND/OR/NOT — websearch_to_tsquery handles them case-insensitively, + // but convert NOT to - (the supported exclusion syntax) + q = q.replaceAll(/\bNOT\b/, '-') + + // Remove wildcards at end of terms (partial matching not directly supported; term will still match as prefix via FTS) + q = q.replaceAll(/\*/, '') + q = q.replaceAll(/\?/, '') + + // Remove empty parentheses, normalize spaces + q = q.replaceAll(/\(\s*\)/, '') + q = q.replaceAll(/\s+/, ' ').trim() + + return q ?: '' + } + + /** + * Build a ts_headline SQL expression for a given field with the given tsquery expression. + * @param fieldJsonPath The SQL expression to extract the text field (e.g. "document->>'productName'") + * @param tsqueryParam The SQL tsquery expression (e.g. "websearch_to_tsquery('english', ?)") + */ + static String buildHighlightExpr(String fieldJsonPath, String tsqueryExpr) { + return "ts_headline('english', coalesce(${fieldJsonPath}, ''), ${tsqueryExpr}, 'StartSel=,StopSel=,MaxWords=35,MinWords=15,ShortWord=3,HighlightAll=false,MaxFragments=3,FragmentDelimiter= ... ')" + } +} diff --git a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy new file mode 100644 index 000000000..a28f247f2 --- /dev/null +++ b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy @@ -0,0 +1,1176 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.impl.context + +import com.fasterxml.jackson.databind.ObjectMapper +import groovy.transform.CompileStatic +import org.moqui.BaseException +import org.moqui.context.ElasticFacade +import org.moqui.entity.EntityValue +import org.moqui.entity.EntityList +import org.moqui.util.MNode +import org.moqui.util.RestClient +import org.moqui.util.RestClient.Method +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.Statement +import java.sql.Timestamp +import java.sql.Types +import java.util.concurrent.Future + +/** + * PostgreSQL-backed implementation of ElasticFacade.ElasticClient. + * + * Stores and searches documents using: + * - moqui_search_index table — tracks index metadata (replaces ES index/alias management) + * - moqui_document table — stores documents as JSONB with tsvector for full-text search + * + * All ElasticSearch Query DSL is translated to PostgreSQL SQL by ElasticQueryTranslator. + * Application logs go to moqui_logs table; HTTP request logs go to moqui_http_log table. + * + * Configured via MoquiConf.xml elastic-facade.cluster with type="postgres". + * Example: + * <cluster name="default" type="postgres" url="transactional" index-prefix="mq_"/> + */ +@CompileStatic +class PostgresElasticClient implements ElasticFacade.ElasticClient { + private final static Logger logger = LoggerFactory.getLogger(PostgresElasticClient.class) + private final static Set DOC_META_KEYS = new HashSet<>(["_index", "_type", "_id", "_timestamp"]) + + /** Jackson mapper shared with ElasticFacadeImpl */ + static final ObjectMapper jacksonMapper = ElasticFacadeImpl.jacksonMapper + + /** Shared UPSERT SQL for moqui_document — used by upsertDocument(), bulkIndex(), and bulkIndexDataDocument() */ + static final String DOCUMENT_UPSERT_SQL = """ + INSERT INTO moqui_document (index_name, doc_id, doc_type, document, content_text, content_tsv, updated_stamp) + VALUES (?, ?, ?, ?::jsonb, ?, to_tsvector('english', COALESCE(?, '')), now()) + ON CONFLICT (index_name, doc_id) DO UPDATE SET + doc_type = EXCLUDED.doc_type, + document = EXCLUDED.document, + content_text = EXCLUDED.content_text, + content_tsv = EXCLUDED.content_tsv, + updated_stamp = EXCLUDED.updated_stamp + """.trim() + + private final ExecutionContextFactoryImpl ecfi + private final MNode clusterNode + private final String clusterName + private final String indexPrefix + /** Entity datasource group to get connections from (e.g. "transactional") */ + final String datasourceGroup + + PostgresElasticClient(MNode clusterNode, ExecutionContextFactoryImpl ecfi) { + this.ecfi = ecfi + this.clusterNode = clusterNode + this.clusterName = clusterNode.attribute("name") + this.indexPrefix = clusterNode.attribute("index-prefix") ?: "" + + // url attribute for postgres type = datasource group name (or "transactional" by default) + String urlAttr = clusterNode.attribute("url") + this.datasourceGroup = (urlAttr && !"".equals(urlAttr.trim())) ? urlAttr.trim() : "transactional" + + logger.info("Initializing PostgresElasticClient for cluster '${clusterName}' using datasource group '${datasourceGroup}' with index prefix '${indexPrefix}'") + + // Initialize schema (CREATE TABLE IF NOT EXISTS, extensions, indexes) + initSchema() + } + + void destroy() { + // Nothing to destroy — connection pool is managed by the entity facade datasource + } + + // ============================================================ + // Schema initialization + // ============================================================ + + private void initSchema() { + boolean started = ecfi.transactionFacade.begin(null) + try { + Connection conn = ecfi.entityFacade.getConnection(datasourceGroup) + Statement stmt = conn.createStatement() + try { + // Enable pg_trgm extension for fuzzy search (available since PG 9.1) + try { stmt.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") } + catch (Exception extEx) { logger.warn("Could not create pg_trgm extension (may require superuser): ${extEx.message}") } + + // moqui_search_index — index metadata (replaces ES index/alias concept) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_search_index ( + index_name TEXT NOT NULL, + alias_name TEXT, + doc_type TEXT, + mapping TEXT, + settings TEXT, + created_stamp TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT pk_moqui_search_index PRIMARY KEY (index_name) + ) + """.trim()) + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_sidx_alias ON moqui_search_index (alias_name)") + + // moqui_document — main document store + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_document ( + index_name TEXT NOT NULL, + doc_id TEXT NOT NULL, + doc_type TEXT, + document JSONB, + content_text TEXT, + content_tsv TSVECTOR, + created_stamp TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_stamp TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT pk_moqui_document PRIMARY KEY (index_name, doc_id) + ) + """.trim()) + // Ensure PostgreSQL-specific columns exist (table may have been created by Moqui entity sync without them) + stmt.execute("ALTER TABLE moqui_document ADD COLUMN IF NOT EXISTS content_tsv TSVECTOR") + stmt.execute("ALTER TABLE moqui_document ADD COLUMN IF NOT EXISTS content_text TEXT") + // Ensure document column is JSONB (entity sync may create it as TEXT from text-very-long mapping) + try { + stmt.execute("ALTER TABLE moqui_document ALTER COLUMN document TYPE JSONB USING document::jsonb") + } catch (Exception e) { + // Column already JSONB or table has no rows causing cast to fail — ignore + logger.trace("Note: could not alter document column to JSONB (may already be correct type): " + e.getMessage()) + } + // GIN index on tsvector for full-text search + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_tsv ON moqui_document USING GIN (content_tsv)") + // GIN index on document JSONB for arbitrary path queries + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_json ON moqui_document USING GIN (document jsonb_path_ops)") + // GIN trigram index on content_text for fuzzy/LIKE queries + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_trgm ON moqui_document USING GIN (content_text gin_trgm_ops)") + // Index for type-based filtering + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_type ON moqui_document (doc_type)") + // Index for time-based ordering + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_upd ON moqui_document (index_name, updated_stamp)") + + // moqui_logs — application log (replaces ES moqui_logs index) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_logs ( + log_id BIGSERIAL PRIMARY KEY, + log_timestamp TIMESTAMPTZ NOT NULL, + log_level TEXT, + thread_name TEXT, + thread_id BIGINT, + thread_priority INTEGER, + logger_name TEXT, + message TEXT, + source_host TEXT, + user_id TEXT, + visitor_id TEXT, + mdc JSONB, + thrown JSONB + ) + """.trim()) + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_logs_ts ON moqui_logs USING BRIN (log_timestamp)") + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_logs_lvl ON moqui_logs (log_level)") + // Fix log_id if Moqui entity sync created the table without a BIGSERIAL default + stmt.execute(""" + DO \$\$ + BEGIN + IF (SELECT column_default FROM information_schema.columns + WHERE table_name = 'moqui_logs' AND column_name = 'log_id') IS NULL THEN + CREATE SEQUENCE IF NOT EXISTS moqui_logs_log_id_seq; + ALTER TABLE moqui_logs ALTER COLUMN log_id SET DEFAULT nextval('moqui_logs_log_id_seq'); + ALTER SEQUENCE moqui_logs_log_id_seq OWNED BY moqui_logs.log_id; + END IF; + END \$\$; + """.trim()) + + // moqui_http_log — HTTP request log (replaces ES moqui_http_log index) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_http_log ( + log_id BIGSERIAL PRIMARY KEY, + log_timestamp TIMESTAMPTZ NOT NULL, + remote_ip TEXT, + remote_user TEXT, + server_ip TEXT, + content_type TEXT, + request_method TEXT, + request_scheme TEXT, + request_host TEXT, + request_path TEXT, + request_query TEXT, + http_version TEXT, + response_code INTEGER, + time_initial_ms BIGINT, + time_final_ms BIGINT, + bytes_sent BIGINT, + referrer TEXT, + agent TEXT, + session_id TEXT, + visitor_id TEXT + ) + """.trim()) + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_hlog_ts ON moqui_http_log USING BRIN (log_timestamp)") + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_hlog_path ON moqui_http_log (request_path)") + // Fix log_id if Moqui entity sync created the table without a BIGSERIAL default + stmt.execute(""" + DO \$\$ + BEGIN + IF (SELECT column_default FROM information_schema.columns + WHERE table_name = 'moqui_http_log' AND column_name = 'log_id') IS NULL THEN + CREATE SEQUENCE IF NOT EXISTS moqui_http_log_log_id_seq; + ALTER TABLE moqui_http_log ALTER COLUMN log_id SET DEFAULT nextval('moqui_http_log_log_id_seq'); + ALTER SEQUENCE moqui_http_log_log_id_seq OWNED BY moqui_http_log.log_id; + END IF; + END \$\$; + """.trim()) + + logger.info("PostgresElasticClient schema initialized for cluster '${clusterName}'") + } finally { + stmt.close() + } + ecfi.transactionFacade.commit(started) + } catch (Throwable t) { + ecfi.transactionFacade.rollback(started, "Error initializing PostgresElasticClient schema", t) + throw new BaseException("Error initializing PostgresElasticClient schema for cluster '${clusterName}'", t) + } + } + + /** + * Get a JDBC Connection from the entity facade for the configured datasource group. + * The returned Connection is a Moqui ConnectionWrapper that is transaction-managed. + */ + private Connection getConnection() { + return ecfi.entityFacade.getConnection(datasourceGroup) + } + + // ============================================================ + // ElasticClient — Cluster info + // ============================================================ + + @Override String getClusterName() { return clusterName } + @Override String getClusterLocation() { return "postgres:${datasourceGroup}:${indexPrefix}" } + + @Override + Map getServerInfo() { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT version()") + try { + ResultSet rs = ps.executeQuery() + try { + if (rs.next()) { + return [name: clusterName, cluster_name: "postgres", + version: [distribution: "postgres", number: rs.getString(1)], + tagline: "Moqui PostgresElasticClient"] + } + } finally { rs.close() } + } finally { ps.close() } + return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres"]] + } + + // ============================================================ + // Index management + // ============================================================ + + @Override + boolean indexExists(String index) { + if (!index) return false + String prefixed = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT 1 FROM moqui_search_index WHERE index_name = ? OR alias_name = ?") + try { + ps.setString(1, prefixed) + ps.setString(2, prefixed) + ResultSet rs = ps.executeQuery() + try { return rs.next() } finally { rs.close() } + } finally { ps.close() } + } + + @Override + boolean aliasExists(String alias) { + if (!alias) return false + String prefixed = prefixIndexName(alias) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT 1 FROM moqui_search_index WHERE alias_name = ?") + try { + ps.setString(1, prefixed) + ResultSet rs = ps.executeQuery() + try { return rs.next() } finally { rs.close() } + } finally { ps.close() } + } + + @Override + void createIndex(String index, Map docMapping, String alias) { + createIndex(index, null, docMapping, alias, null) + } + + void createIndex(String index, String docType, Map docMapping, String alias, Map settings) { + if (!index) throw new IllegalArgumentException("Index name required for createIndex") + String prefixedIndex = prefixIndexName(index) + String prefixedAlias = alias ? prefixIndexName(alias) : null + + String mappingJson = docMapping ? objectToJson(docMapping) : null + String settingsJson = settings ? objectToJson(settings) : null + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(""" + INSERT INTO moqui_search_index (index_name, alias_name, doc_type, mapping, settings) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (index_name) DO UPDATE SET + alias_name = EXCLUDED.alias_name, + doc_type = EXCLUDED.doc_type, + mapping = EXCLUDED.mapping, + settings = EXCLUDED.settings + """.trim()) + try { + ps.setString(1, prefixedIndex) + if (prefixedAlias) ps.setString(2, prefixedAlias) else ps.setNull(2, Types.VARCHAR) + if (docType) ps.setString(3, docType) else ps.setNull(3, Types.VARCHAR) + if (mappingJson) ps.setString(4, mappingJson) else ps.setNull(4, Types.VARCHAR) + if (settingsJson) ps.setString(5, settingsJson) else ps.setNull(5, Types.VARCHAR) + ps.executeUpdate() + } finally { ps.close() } + logger.info("PostgresElasticClient.createIndex: created index '${prefixedIndex}'${prefixedAlias ? ' with alias ' + prefixedAlias : ''}") + } + + @Override + void putMapping(String index, Map docMapping) { + if (!docMapping) throw new IllegalArgumentException("Mapping may not be empty for putMapping") + String prefixedIndex = prefixIndexName(index) + String mappingJson = objectToJson(docMapping) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "UPDATE moqui_search_index SET mapping = ? WHERE index_name = ?") + try { + ps.setString(1, mappingJson) + ps.setString(2, prefixedIndex) + ps.executeUpdate() + } finally { ps.close() } + } + + @Override + void deleteIndex(String index) { + if (!index) throw new IllegalArgumentException("Index name required for deleteIndex") + String prefixedIndex = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps1 = conn.prepareStatement("DELETE FROM moqui_document WHERE index_name = ?") + try { + ps1.setString(1, prefixedIndex) + int deleted = ps1.executeUpdate() + logger.info("PostgresElasticClient.deleteIndex: deleted ${deleted} documents from index '${prefixedIndex}'") + } finally { ps1.close() } + PreparedStatement ps2 = conn.prepareStatement("DELETE FROM moqui_search_index WHERE index_name = ?") + try { + ps2.setString(1, prefixedIndex) + ps2.executeUpdate() + } finally { ps2.close() } + } + + // ============================================================ + // Document CRUD + // ============================================================ + + @Override + void index(String index, String _id, Map document) { + if (!index) throw new IllegalArgumentException("Index name required for index()") + if (!_id) throw new IllegalArgumentException("_id required for index()") + String prefixedIndex = prefixIndexName(index) + String docJson = objectToJson(document) + String contentText = extractContentText(document) + upsertDocument(prefixedIndex, _id, null, docJson, contentText) + } + + @Override + void update(String index, String _id, Map documentFragment) { + if (!index) throw new IllegalArgumentException("Index name required for update()") + if (!_id) throw new IllegalArgumentException("_id required for update()") + String prefixedIndex = prefixIndexName(index) + String fragmentJson = objectToJson(documentFragment) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(""" + UPDATE moqui_document + SET document = COALESCE(document, '{}'::jsonb) || ?::jsonb, + content_text = ( + SELECT string_agg(val::text, ' ') + FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) + WHERE jsonb_typeof(COALESCE(document, '{}'::jsonb) || ?::jsonb -> kv.key) IN ('string', 'number') + ), + content_tsv = to_tsvector('english', coalesce(( + SELECT string_agg(val::text, ' ') + FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) + ), '')), + updated_stamp = now() + WHERE index_name = ? AND doc_id = ? + """.trim()) + try { + ps.setString(1, fragmentJson) + ps.setString(2, fragmentJson) + ps.setString(3, fragmentJson) + ps.setString(4, fragmentJson) + ps.setString(5, prefixedIndex) + ps.setString(6, _id) + ps.executeUpdate() + } finally { ps.close() } + } + + @Override + void delete(String index, String _id) { + if (!index) throw new IllegalArgumentException("Index name required for delete()") + if (!_id) throw new IllegalArgumentException("_id required for delete()") + String prefixedIndex = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM moqui_document WHERE index_name = ? AND doc_id = ?") + try { + ps.setString(1, prefixedIndex) + ps.setString(2, _id) + int deleted = ps.executeUpdate() + if (deleted == 0) logger.warn("delete() document not found in index '${prefixedIndex}' with id '${_id}'") + } finally { ps.close() } + } + + @Override + Integer deleteByQuery(String index, Map queryMap) { + if (!index) throw new IllegalArgumentException("Index name required for deleteByQuery()") + String prefixedIndex = prefixIndexName(index) + ElasticQueryTranslator.QueryResult qr = ElasticQueryTranslator.translateQuery(queryMap ?: [match_all: [:]]) + + List allParams = new ArrayList<>() + allParams.add(prefixedIndex) + if (qr.params) allParams.addAll(qr.params) + String sql = "DELETE FROM moqui_document WHERE index_name = ? AND (${qr.clause})" + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < allParams.size(); i++) { + setParam(ps, i + 1, allParams[i]) + } + return ps.executeUpdate() + } finally { ps.close() } + } + + @Override + void bulk(String index, List actionSourceList) { + if (!actionSourceList) return + String prefixedIndex = index ? prefixIndexName(index) : null + + int i = 0 + while (i < actionSourceList.size()) { + Map action = (Map) actionSourceList.get(i) + + if (action.containsKey("delete")) { + Map actionSpec = (Map) action.get("delete") + String idxName = actionSpec.get("_index") ? prefixIndexName((String) actionSpec.get("_index")) : prefixedIndex + String _id = (String) actionSpec.get("_id") + if (idxName && _id) delete(idxName, _id) + i += 1 + } else if (i + 1 < actionSourceList.size()) { + Map source = (Map) actionSourceList.get(i + 1) + + if (action.containsKey("index") || action.containsKey("create")) { + Map actionSpec = (Map) (action.get("index") ?: action.get("create")) + String idxName = actionSpec.get("_index") ? prefixIndexName((String) actionSpec.get("_index")) : prefixedIndex + String _id = (String) actionSpec.get("_id") + if (idxName) { + String docJson = objectToJson(source) + String contentText = extractContentText(source) + upsertDocument(idxName, _id, null, docJson, contentText) + } + } else if (action.containsKey("update")) { + Map actionSpec = (Map) action.get("update") + String idxName = actionSpec.get("_index") ? prefixIndexName((String) actionSpec.get("_index")) : prefixedIndex + String _id = (String) actionSpec.get("_id") + if (idxName && _id) { + Map doc = (Map) source.get("doc") ?: source + update(idxName, _id, doc) + } + } + i += 2 + } else { + logger.warn("bulk(): action at index ${i} has no following source document, skipping") + i += 1 + } + } + } + + @Override + void bulkIndex(String index, String idField, List documentList) { + bulkIndex(index, null, idField, documentList, false) + } + + @Override + void bulkIndex(String index, String docType, String idField, List documentList, boolean refresh) { + if (!documentList) return + String prefixedIndex = prefixIndexName(index) + boolean hasId = idField != null && !idField.isEmpty() + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(DOCUMENT_UPSERT_SQL) + try { + int batchSize = 0 + for (Map doc in documentList) { + String _id = hasId ? (doc.get(idField)?.toString() ?: UUID.randomUUID().toString()) : UUID.randomUUID().toString() + String docJson = objectToJson(doc) + String contentText = extractContentText(doc) + setUpsertParams(ps, prefixedIndex, _id, docType, docJson, contentText) + ps.addBatch() + batchSize++ + if (batchSize >= 500) { + ps.executeBatch() + batchSize = 0 + } + } + if (batchSize > 0) ps.executeBatch() + } finally { ps.close() } + } + + @Override + Map get(String index, String _id) { + if (!index || !_id) return null + String prefixedIndex = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT doc_id, index_name, doc_type, document FROM moqui_document WHERE index_name = ? AND doc_id = ?") + try { + ps.setString(1, prefixedIndex) + ps.setString(2, _id) + ResultSet rs = ps.executeQuery() + try { + if (rs.next()) { + Map source = (Map) jsonToObject(rs.getString("document")) + return [_index: unprefixIndexName(rs.getString("index_name")), + _id : rs.getString("doc_id"), + _type : rs.getString("doc_type"), + _source: source] + } + return null + } finally { rs.close() } + } finally { ps.close() } + } + + @Override + Map getSource(String index, String _id) { + Map result = get(index, _id) + return result ? (Map) result.get("_source") : null + } + + @Override + List get(String index, List _idList) { + if (!_idList || !index) return [] + String prefixedIndex = prefixIndexName(index) + String placeholders = _idList.collect { "?" }.join(", ") + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT doc_id, index_name, doc_type, document FROM moqui_document WHERE index_name = ? AND doc_id IN (${placeholders})") + try { + ps.setString(1, prefixedIndex) + for (int i = 0; i < _idList.size(); i++) ps.setString(i + 2, _idList[i]) + ResultSet rs = ps.executeQuery() + try { + List results = [] + while (rs.next()) { + Map source = (Map) jsonToObject(rs.getString("document")) + results.add([_index: unprefixIndexName(rs.getString("index_name")), + _id : rs.getString("doc_id"), + _type : rs.getString("doc_type"), + _source: source]) + } + return results + } finally { rs.close() } + } finally { ps.close() } + } + + // ============================================================ + // Search + // ============================================================ + + @Override + Map search(String index, Map searchMap) { + if (index && (index == 'moqui_logs' || prefixIndexName(index) == prefixIndexName('moqui_logs'))) { + return searchLogsTable(searchMap ?: [:]) + } + + ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap ?: [:]) + + List indexNames = resolveIndexNames(index) + if (indexNames.isEmpty()) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + + String scoreExpr = tq.tsqueryExpr ? + "ts_rank_cd(content_tsv, ${tq.tsqueryExpr})" : "1.0::float" + + String idxPlaceholders = indexNames.collect { "?" }.join(", ") + String whereClause = "index_name IN (${idxPlaceholders})" + List allParams = new ArrayList<>(indexNames) + + if (tq.tsqueryExpr) { + whereClause += " AND " + tq.whereClause + allParams.addAll(tq.params) + } else if (tq.whereClause && tq.whereClause != "TRUE") { + whereClause += " AND " + tq.whereClause + allParams.addAll(tq.params) + } + + String orderByClause = tq.orderBy ?: (tq.tsqueryExpr ? "_score DESC" : "updated_stamp DESC") + + String countSql = "SELECT COUNT(*) FROM moqui_document WHERE ${whereClause}" + long totalCount = 0L + + String mainSql = """ + SELECT doc_id, index_name, doc_type, document, ${buildScoreSelect(tq)} AS _score + FROM moqui_document + WHERE ${whereClause} + ORDER BY ${orderByClause} + LIMIT ? OFFSET ? + """.trim() + + Connection conn = getConnection() + + if (tq.trackTotal) { + PreparedStatement countPs = conn.prepareStatement(countSql) + try { + for (int i = 0; i < allParams.size(); i++) setParam(countPs, i + 1, allParams[i]) + ResultSet rs = countPs.executeQuery() + try { if (rs.next()) totalCount = rs.getLong(1) } finally { rs.close() } + } finally { countPs.close() } + } + + List mainParams = [] + if (tq.tsqueryExpr) mainParams.addAll(tq.tsqueryParams) + mainParams.addAll(allParams) + mainParams.add(tq.sizeLimit) + mainParams.add(tq.fromOffset) + + PreparedStatement ps = conn.prepareStatement(mainSql) + try { + for (int i = 0; i < mainParams.size(); i++) setParam(ps, i + 1, mainParams[i]) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + String docJson = rs.getString("document") + Map source = docJson ? (Map) jsonToObject(docJson) : [:] + String docId = rs.getString("doc_id") + String idxName = unprefixIndexName(rs.getString("index_name")) + String docType = rs.getString("doc_type") + double score = rs.getDouble("_score") + + Map hit = [_index: idxName, _id: docId, _type: docType, + _score: score, _source: source] as Map + + if (tq.highlightFields && tq.tsqueryExpr) { + Map> highlights = buildHighlights(source, tq) + if (highlights) hit.put("highlight", highlights) + } + + hits.add(hit) + } + + return [hits: [total: [value: totalCount, relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } finally { ps.close() } + } + + private Map searchLogsTable(Map searchMap) { + ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap) + + String rawQuery = null + Map queryMap = (Map) searchMap?.get("query") + if (queryMap) { + Map qsMap = (Map) queryMap.get("query_string") + if (qsMap) rawQuery = (String) qsMap.get("query") + } + + List conditions = [] + List params = [] + + if (rawQuery) { + java.util.regex.Matcher m = (rawQuery =~ /@timestamp\s*:\s*\[\s*(\*|\d+)\s+TO\s+(\*|\d+)\s*\]/) + if (m.find()) { + String fromVal = m.group(1) + String toVal = m.group(2) + if (fromVal != '*') { + conditions.add("log_timestamp >= ?") + params.add(new java.sql.Timestamp(Long.parseLong(fromVal))) + } + if (toVal != '*') { + conditions.add("log_timestamp <= ?") + params.add(new java.sql.Timestamp(Long.parseLong(toVal))) + } + } + } + + String userTextQuery = null + if (rawQuery) { + String stripped = rawQuery.replaceAll(/@timestamp\s*:\s*\[[^\]]*\]/, '') + stripped = stripped.replaceAll(/\bAND\b/, ' ').replaceAll(/\bOR\b/, ' ') + stripped = stripped.replaceAll(/[()]/, ' ').replaceAll(/\s+/, ' ').trim() + stripped = stripped.replaceAll(/\*/, '').trim() + if (stripped) userTextQuery = stripped + } + if (userTextQuery) { + conditions.add("to_tsvector('english', coalesce(message, '') || ' ' || coalesce(logger_name, '')) @@ websearch_to_tsquery('english', ?)") + params.add(userTextQuery) + } + + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + + Connection conn = getConnection() + long totalCount = 0L + if (tq.trackTotal) { + PreparedStatement countPs = conn.prepareStatement("SELECT COUNT(*) FROM moqui_logs WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(countPs, i + 1, params[i]) + ResultSet rs = countPs.executeQuery() + try { if (rs.next()) totalCount = rs.getLong(1) } finally { rs.close() } + } finally { countPs.close() } + } + + String mainSql = """ + SELECT log_id, log_timestamp, log_level, thread_name, thread_id, thread_priority, + logger_name, message, source_host, user_id, visitor_id, mdc::text, thrown::text + FROM moqui_logs + WHERE ${whereClause} + ORDER BY log_timestamp DESC + LIMIT ? OFFSET ? + """.trim() + + PreparedStatement ps = conn.prepareStatement(mainSql) + try { + int pIdx = 0 + for (int i = 0; i < params.size(); i++) setParam(ps, ++pIdx, params[i]) + ps.setInt(++pIdx, tq.sizeLimit) + ps.setInt(++pIdx, tq.fromOffset) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + long logId = rs.getLong("log_id") + java.sql.Timestamp ts = rs.getTimestamp("log_timestamp") + Map source = [ + "@timestamp" : ts?.time, + level : rs.getString("log_level"), + thread_name : rs.getString("thread_name"), + thread_id : rs.getLong("thread_id"), + thread_priority : rs.getInt("thread_priority"), + logger_name : rs.getString("logger_name"), + message : rs.getString("message"), + source_host : rs.getString("source_host"), + user_id : rs.getString("user_id"), + visitor_id : rs.getString("visitor_id"), + ] as Map + String mdcStr = rs.getString("mdc") + if (mdcStr) source.put("mdc", jsonToObject(mdcStr)) + String thrownStr = rs.getString("thrown") + if (thrownStr) source.put("thrown", jsonToObject(thrownStr)) + hits.add([_index: "moqui_logs", _id: String.valueOf(logId), + _type: "LogMessage", _score: 1.0, _source: source] as Map) + } + return [hits: [total: [value: totalCount, relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } catch (Throwable t) { + logger.error("searchLogsTable error: " + t.message, t) + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } finally { ps.close() } + } + + @Override + List searchHits(String index, Map searchMap) { + Map result = search(index, searchMap) + return (List) ((Map) result.get("hits")).get("hits") + } + + @Override + Map validateQuery(String index, Map queryMap, boolean explain) { + try { + ElasticQueryTranslator.QueryResult qr = ElasticQueryTranslator.translateQuery(queryMap ?: [match_all: [:]]) + return null // valid + } catch (Throwable t) { + return [valid: false, error: t.message] + } + } + + @Override + long count(String index, Map countMap) { + Map result = countResponse(index, countMap) + return ((Number) result.get("count"))?.longValue() ?: 0L + } + + @Override + Map countResponse(String index, Map countMap) { + if (!countMap) countMap = [query: [match_all: [:]]] + Map queryMap = (Map) countMap.get("query") + ElasticQueryTranslator.QueryResult qr = queryMap ? ElasticQueryTranslator.translateQuery(queryMap) : new ElasticQueryTranslator.QueryResult() + + List indexNames = resolveIndexNames(index) + if (indexNames.isEmpty()) return [count: 0L] + + String idxPlaceholders = indexNames.collect { "?" }.join(", ") + String whereClause = "index_name IN (${idxPlaceholders})" + List allParams = new ArrayList<>(indexNames) + + if (qr.clause && qr.clause != "TRUE") { + whereClause += " AND " + qr.clause + allParams.addAll(qr.params) + } + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT COUNT(*) FROM moqui_document WHERE ${whereClause}") + try { + for (int i = 0; i < allParams.size(); i++) setParam(ps, i + 1, allParams[i]) + ResultSet rs = ps.executeQuery() + try { + if (rs.next()) return [count: rs.getLong(1)] + return [count: 0L] + } finally { rs.close() } + } finally { ps.close() } + } + + // ============================================================ + // Point-In-Time (PIT) — Keyset-based cursor + // ============================================================ + + @Override + String getPitId(String index, String keepAlive) { + return "pg::${indexPrefix}::${System.currentTimeMillis()}" + } + + @Override + void deletePit(String pitId) { + // No-op for postgres backend + } + + // ============================================================ + // Raw REST — Not supported on postgres backend + // ============================================================ + + @Override + RestClient.RestResponse call(Method method, String index, String path, + Map parameters, Object bodyJsonObject) { + throw new UnsupportedOperationException( + "Raw REST calls (call()) are not supported by PostgresElasticClient for cluster '${clusterName}'. " + + "Use the higher-level API methods instead, or switch to type=elastic for this cluster.") + } + + @Override + Future callFuture(Method method, String index, String path, + Map parameters, Object bodyJsonObject) { + throw new UnsupportedOperationException( + "Raw REST calls (callFuture()) are not supported by PostgresElasticClient for cluster '${clusterName}'.") + } + + @Override + RestClient makeRestClient(Method method, String index, String path, Map parameters) { + throw new UnsupportedOperationException( + "makeRestClient() is not supported by PostgresElasticClient for cluster '${clusterName}'.") + } + + // ============================================================ + // DataDocument helpers + // ============================================================ + + @Override + void checkCreateDataDocumentIndexes(String indexName) { + if (!indexName) return + if (indexExists(indexName)) return + EntityList ddList = ecfi.entityFacade.find("moqui.entity.document.DataDocument") + .condition("indexName", indexName).disableAuthz().list() + for (EntityValue dd in ddList) { + storeIndexAndMapping(indexName, dd) + } + } + + @Override + void checkCreateDataDocumentIndex(String dataDocumentId) { + String idxName = ElasticFacadeImpl.ddIdToEsIndex(dataDocumentId) + String prefixed = prefixIndexName(idxName) + if (indexExists(prefixed)) return + + EntityValue dd = ecfi.entityFacade.find("moqui.entity.document.DataDocument") + .condition("dataDocumentId", dataDocumentId).disableAuthz().one() + if (dd == null) throw new BaseException("No DataDocument found with ID [${dataDocumentId}]") + storeIndexAndMapping((String) dd.getNoCheckSimple("indexName"), dd) + } + + @Override + void putDataDocumentMappings(String indexName) { + EntityList ddList = ecfi.entityFacade.find("moqui.entity.document.DataDocument") + .condition("indexName", indexName).disableAuthz().list() + for (EntityValue dd in ddList) storeIndexAndMapping(indexName, dd) + } + + @Override + void verifyDataDocumentIndexes(List documentList) { + Set indexNames = new HashSet<>() + Set dataDocumentIds = new HashSet<>() + for (Map doc in documentList) { + Object idxObj = doc.get("_index") + Object typeObj = doc.get("_type") + if (idxObj) indexNames.add((String) idxObj) + if (typeObj) dataDocumentIds.add((String) typeObj) + } + for (String idxName in indexNames) checkCreateDataDocumentIndexes(idxName) + for (String ddId in dataDocumentIds) checkCreateDataDocumentIndex(ddId) + } + + @Override + void bulkIndexDataDocument(List documentList) { + if (!documentList) return + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(DOCUMENT_UPSERT_SQL) + try { + int batchCount = 0 + for (Map document in documentList) { + String _index = (String) document.get("_index") + String _type = (String) document.get("_type") + String _id = (String) document.get("_id") + + if (!_id) { + logger.warn("bulkIndexDataDocument: skipping document with null _id (type=${_type})") + continue + } + + String esIndexName = ElasticFacadeImpl.ddIdToEsIndex(_type ?: "unknown") + String prefixedIndex = prefixIndexName(esIndexName) + + Map cleanDoc = new LinkedHashMap<>(document) + for (String key in DOC_META_KEYS) cleanDoc.remove(key) + + String docJson = objectToJson(cleanDoc) + String contentText = extractContentText(cleanDoc) + + setUpsertParams(ps, prefixedIndex, _id, _type, docJson, contentText) + ps.addBatch() + batchCount++ + + if (batchCount >= 500) { + ps.executeBatch() + batchCount = 0 + } + } + if (batchCount > 0) ps.executeBatch() + logger.info("bulkIndexDataDocument: indexed ${documentList.size()} documents") + } finally { ps.close() } + } + + // ============================================================ + // JSON serialization + // ============================================================ + + @Override String objectToJson(Object obj) { return ElasticFacadeImpl.objectToJson(obj) } + @Override Object jsonToObject(String json) { return ElasticFacadeImpl.jsonToObject(json) } + + // ============================================================ + // Index prefixing helpers + // ============================================================ + + String prefixIndexName(String index) { + if (!index) return index + index = index.trim() + if (!index) return index + return index.split(",").collect { String it -> + it = it.trim() + return (indexPrefix && !it.startsWith(indexPrefix)) ? indexPrefix + it : it + }.join(",") + } + + String unprefixIndexName(String index) { + if (!index || !indexPrefix) return index + index = index.trim() + return index.split(",").collect { String it -> + it = it.trim() + return (indexPrefix && it.startsWith(indexPrefix)) ? it.substring(indexPrefix.length()) : it + }.join(",") + } + + // ============================================================ + // Private helpers + // ============================================================ + + /** Set the 6 parameters on a PreparedStatement using the shared DOCUMENT_UPSERT_SQL */ + private static void setUpsertParams(PreparedStatement ps, String prefixedIndex, String docId, String docType, String docJson, String contentText) { + ps.setString(1, prefixedIndex) + ps.setString(2, docId ?: UUID.randomUUID().toString()) + if (docType) ps.setString(3, docType) else ps.setNull(3, Types.VARCHAR) + ps.setString(4, docJson) + ps.setString(5, contentText) + ps.setString(6, contentText) + } + + private void upsertDocument(String prefixedIndex, String docId, String docType, String docJson, String contentText) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(DOCUMENT_UPSERT_SQL) + try { + setUpsertParams(ps, prefixedIndex, docId, docType, docJson, contentText) + ps.executeUpdate() + } finally { ps.close() } + } + + static String extractContentText(Map document) { + if (document == null || document.isEmpty()) return "" + StringBuilder sb = new StringBuilder() + extractTextFromValue(document, sb) + return sb.toString().trim() + } + + private static void extractTextFromValue(Object value, StringBuilder sb) { + if (value instanceof Map) { + for (Map.Entry entry in ((Map) value).entrySet()) { + Object k = entry.key + Object v = entry.value + if (k instanceof String) { + String key = (String) k + if (!key.endsWith("Id") || key.length() < 20) { + extractTextFromValue(v, sb) + } + } else { + extractTextFromValue(v, sb) + } + } + } else if (value instanceof List) { + for (Object item in (List) value) extractTextFromValue(item, sb) + } else if (value instanceof String) { + String s = (String) value + if (s.length() > 0) { + if (sb.length() > 0) sb.append(' ') + sb.append(s) + } + } else if (value instanceof Number || value instanceof Boolean) { + if (sb.length() > 0) sb.append(' ') + sb.append(value.toString()) + } + } + + protected synchronized void storeIndexAndMapping(String indexName, EntityValue dd) { + String dataDocumentId = (String) dd.getNoCheckSimple("dataDocumentId") + String manualMappingServiceName = (String) dd.getNoCheckSimple("manualMappingServiceName") + String esIndexName = ElasticFacadeImpl.ddIdToEsIndex(dataDocumentId) + String prefixedIndex = prefixIndexName(esIndexName) + + boolean hasIndex = indexExists(prefixedIndex) + Map docMapping = ElasticFacadeImpl.makeElasticSearchMapping(dataDocumentId, ecfi) + Map settings = null + + if (manualMappingServiceName) { + Map serviceResult = ecfi.service.sync().name(manualMappingServiceName) + .parameter("mapping", docMapping).call() + docMapping = (Map) serviceResult.get("mapping") + settings = (Map) serviceResult.get("settings") + } + + if (hasIndex) { + logger.info("PostgresElasticClient: updating mapping for index '${prefixedIndex}' (${dataDocumentId})") + putMapping(prefixedIndex, docMapping) + } else { + logger.info("PostgresElasticClient: creating index '${prefixedIndex}' for DataDocument '${dataDocumentId}' with alias '${indexName}'") + createIndex(prefixedIndex, dataDocumentId, docMapping, indexName, settings) + } + } + + private List resolveIndexNames(String index) { + if (!index) { + return getAllIndexNames() + } + List result = [] + for (String part in index.split(",")) { + String trimmed = part.trim() + if (!trimmed) continue + String prefixed = prefixIndexName(trimmed) + List aliasResolved = resolveAlias(prefixed) + if (aliasResolved) { + result.addAll(aliasResolved) + } else { + result.add(prefixed) + } + } + return result + } + + private List resolveAlias(String alias) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT index_name FROM moqui_search_index WHERE alias_name = ?") + try { + ps.setString(1, alias) + ResultSet rs = ps.executeQuery() + try { + List names = [] + while (rs.next()) names.add(rs.getString("index_name")) + return names + } finally { rs.close() } + } finally { ps.close() } + } + + private List getAllIndexNames() { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT index_name FROM moqui_search_index") + try { + ResultSet rs = ps.executeQuery() + try { + List names = [] + while (rs.next()) names.add(rs.getString("index_name")) + return names + } finally { rs.close() } + } finally { ps.close() } + } + + private static String buildScoreSelect(ElasticQueryTranslator.TranslatedQuery tq) { + if (tq.tsqueryExpr) { + return "ts_rank_cd(content_tsv, ${tq.tsqueryExpr})" + } + return "1.0::float" + } + + private static Map> buildHighlights(Map source, ElasticQueryTranslator.TranslatedQuery tq) { + Map> highlights = [:] + if (!tq.tsqueryExpr || !tq.highlightFields) return highlights + String firstParam = tq.params ? tq.params[0]?.toString() : null + if (!firstParam) return highlights + for (String field in tq.highlightFields.keySet()) { + Object fieldVal = source.get(field) + if (fieldVal instanceof String) { + String text = (String) fieldVal + String highlighted = simpleHighlight(text, firstParam) + if (highlighted != text) highlights.put(field, [highlighted]) + } + } + return highlights + } + + private static String simpleHighlight(String text, String query) { + if (!text || !query) return text + List terms = query.replaceAll(/["()+\-]/, ' ').split(/\s+/).findAll { it.length() > 2 } as List + String result = text + for (String term in terms) { + result = result.replaceAll("(?i)\\b${java.util.regex.Pattern.quote(term)}\\b", "\$0") + } + return result + } + + private static void setParam(PreparedStatement ps, int idx, Object value) { + if (value == null) { + ps.setNull(idx, Types.VARCHAR) + } else if (value instanceof String) { + ps.setString(idx, (String) value) + } else if (value instanceof Long || value instanceof Integer) { + ps.setLong(idx, ((Number) value).longValue()) + } else if (value instanceof Double || value instanceof Float || value instanceof BigDecimal) { + ps.setDouble(idx, ((Number) value).doubleValue()) + } else if (value instanceof Timestamp) { + ps.setTimestamp(idx, (Timestamp) value) + } else { + ps.setString(idx, value.toString()) + } + } +} diff --git a/framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy b/framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy new file mode 100644 index 000000000..d300442e9 --- /dev/null +++ b/framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy @@ -0,0 +1,244 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.impl.util + +import groovy.transform.CompileStatic +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.core.LogEvent +import org.apache.logging.log4j.util.ReadOnlyStringMap +import org.moqui.BaseArtifactException +import org.moqui.context.ArtifactExecutionInfo +import org.moqui.context.LogEventSubscriber +import org.moqui.impl.context.ExecutionContextFactoryImpl +import org.moqui.impl.context.PostgresElasticClient +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.Timestamp +import java.sql.Types +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean + +/** + * PostgreSQL-backed application log appender (replaces ElasticSearchLogger for postgres clusters). + * + * Consumes LogEvent objects from a queue and batch-inserts them into the moqui_logs table. + * The queue is flushed every 3 seconds by a scheduled task, identical to ElasticSearchLogger behaviour. + */ +@CompileStatic +class PostgresSearchLogger { + private final static Logger logger = LoggerFactory.getLogger(PostgresSearchLogger.class) + + final static int QUEUE_LIMIT = 16384 + + private final PostgresElasticClient pgClient + private final ExecutionContextFactoryImpl ecfi + + private boolean initialized = false + private volatile boolean disabled = false + + final ConcurrentLinkedQueue logMessageQueue = new ConcurrentLinkedQueue<>() + final AtomicBoolean flushRunning = new AtomicBoolean(false) + + protected PgLogSubscriber subscriber = null + + PostgresSearchLogger(PostgresElasticClient pgClient, ExecutionContextFactoryImpl ecfi) { + this.pgClient = pgClient + this.ecfi = ecfi + init() + } + + void init() { + // moqui_logs table is created by PostgresElasticClient.initSchema() — no extra setup needed + + // Schedule flush every 3 seconds (same cadence as ElasticSearchLogger) + PgLogQueueFlush flushTask = new PgLogQueueFlush(this) + ecfi.scheduleAtFixedRate(flushTask, 10, 3) + + subscriber = new PgLogSubscriber(this) + ecfi.registerLogEventSubscriber(subscriber) + + initialized = true + logger.info("PostgresSearchLogger initialized for cluster '${pgClient.clusterName}'") + } + + void destroy() { disabled = true } + + boolean isInitialized() { return initialized } + + // ============================================================ + // Log subscriber — mirrors ElasticSearchSubscriber + // ============================================================ + + static class PgLogSubscriber implements LogEventSubscriber { + private final PostgresSearchLogger pgLogger + private final InetAddress localAddr = InetAddress.getLocalHost() + + PgLogSubscriber(PostgresSearchLogger pgLogger) { this.pgLogger = pgLogger } + + @Override + void process(LogEvent event) { + if (pgLogger.disabled) return + // Suppress DEBUG / TRACE (same rule as ElasticSearchLogger) + if (Level.DEBUG.is(event.level) || Level.TRACE.is(event.level)) return + // Back-pressure: if queue too full, drop the oldest-style (newest is not enqueued) + if (pgLogger.logMessageQueue.size() >= QUEUE_LIMIT) return + + Map msgMap = [ + '@timestamp' : event.timeMillis, + level : event.level.toString(), + thread_name : event.threadName, + thread_id : event.threadId, + thread_priority: event.threadPriority, + logger_name : event.loggerName, + message : event.message?.formattedMessage, + source_host : localAddr.hostName + ] as Map + + ReadOnlyStringMap contextData = event.contextData + if (contextData != null && contextData.size() > 0) { + Map mdcMap = new HashMap<>(contextData.toMap()) + String userId = mdcMap.remove("moqui_userId") + String visitorId = mdcMap.remove("moqui_visitorId") + if (userId) msgMap.put("user_id", userId) + if (visitorId) msgMap.put("visitor_id", visitorId) + if (mdcMap.size() > 0) msgMap.put("mdc", mdcMap) + } + Throwable thrown = event.thrown + if (thrown != null) msgMap.put("thrown", ElasticSearchLogger.ElasticSearchSubscriber.makeThrowableMap(thrown)) + + pgLogger.logMessageQueue.add(msgMap) + } + } + + // ============================================================ + // Scheduled flush task — drains queue into moqui_logs via JDBC + // ============================================================ + + static class PgLogQueueFlush implements Runnable { + private final static int MAX_BATCH = 200 + + private final PostgresSearchLogger pgLogger + + PgLogQueueFlush(PostgresSearchLogger pgLogger) { this.pgLogger = pgLogger } + + @Override + void run() { + if (!pgLogger.flushRunning.compareAndSet(false, true)) return + try { + while (pgLogger.logMessageQueue.size() > 0) { flushQueue() } + } finally { + pgLogger.flushRunning.set(false) + } + } + + void flushQueue() { + final ConcurrentLinkedQueue queue = pgLogger.logMessageQueue + List batch = new ArrayList<>(MAX_BATCH) + long lastTs = 0L + int sameCount = 0 + + while (batch.size() < MAX_BATCH) { + Map msg = queue.poll() + if (msg == null) break + // Ensure unique timestamps (same as ES logger behaviour) + long ts = (msg.get("@timestamp") as long) ?: System.currentTimeMillis() + if (ts == lastTs) { + sameCount++ + ts += sameCount + msg.put("@timestamp", ts) + } else { + lastTs = ts + sameCount = 0 + } + batch.add(msg) + } + + if (batch.isEmpty()) return + + int retries = 3 + while (retries-- > 0) { + try { + writeBatch(batch) + return + } catch (Throwable t) { + System.out.println("PostgresSearchLogger: error writing log batch, retries left ${retries}: ${t}") + if (retries == 0) System.out.println("PostgresSearchLogger: dropping ${batch.size()} log records after repeated failures") + } + } + } + + private void writeBatch(List batch) { + boolean txStarted = pgLogger.ecfi.transactionFacade.begin(null) + try { + Connection conn = pgLogger.ecfi.entityFacade.getConnection(pgLogger.pgClient.datasourceGroup) + PreparedStatement ps = conn.prepareStatement(""" + INSERT INTO moqui_logs (log_timestamp, log_level, thread_name, thread_id, thread_priority, + logger_name, message, source_host, user_id, visitor_id, mdc, thrown) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?::jsonb) + """.trim()) + try { + for (Map msg in batch) { + // Timestamp: stored as epoch_millis long in the map + long tsMillis = (msg.get("@timestamp") as long) ?: System.currentTimeMillis() + ps.setTimestamp(1, new Timestamp(tsMillis)) + setStr(ps, 2, msg.get("level") as String) + setStr(ps, 3, msg.get("thread_name") as String) + setLong(ps, 4, msg.get("thread_id") as Long) + setInt(ps, 5, msg.get("thread_priority") as Integer) + setStr(ps, 6, msg.get("logger_name") as String) + setStr(ps, 7, msg.get("message") as String) + setStr(ps, 8, msg.get("source_host") as String) + setStr(ps, 9, msg.get("user_id") as String) + setStr(ps, 10, msg.get("visitor_id") as String) + // mdc and thrown as JSONB + Object mdcObj = msg.get("mdc") + setJsonb(ps, 11, mdcObj) + Object thrownObj = msg.get("thrown") + setJsonb(ps, 12, thrownObj) + ps.addBatch() + } + ps.executeBatch() + } finally { ps.close() } + pgLogger.ecfi.transactionFacade.commit(txStarted) + } catch (Throwable t) { + pgLogger.ecfi.transactionFacade.rollback(txStarted, "Error writing log batch to moqui_logs", t) + throw t + } + } + + private static void setStr(PreparedStatement ps, int i, String v) { + if (v == null) ps.setNull(i, Types.VARCHAR) else ps.setString(i, v) + } + private static void setLong(PreparedStatement ps, int i, Long v) { + if (v == null) ps.setNull(i, Types.BIGINT) else ps.setLong(i, v) + } + private static void setInt(PreparedStatement ps, int i, Integer v) { + if (v == null) ps.setNull(i, Types.INTEGER) else ps.setInt(i, v) + } + private static void setJsonb(PreparedStatement ps, int i, Object v) { + if (v == null) { + ps.setNull(i, Types.OTHER) + } else { + try { + ps.setString(i, PostgresElasticClient.jacksonMapper.writeValueAsString(v)) + } catch (Throwable t) { + ps.setNull(i, Types.OTHER) + } + } + } + } +} diff --git a/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy b/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy index 149b66a79..395c606fb 100644 --- a/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy +++ b/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy @@ -15,7 +15,7 @@ package org.moqui.impl.webapp import groovy.transform.CompileStatic import org.moqui.Moqui -import org.moqui.impl.context.ElasticFacadeImpl.ElasticClientImpl +import org.moqui.context.ElasticFacade import org.moqui.impl.context.ExecutionContextFactoryImpl import org.moqui.impl.context.UserFacadeImpl import org.slf4j.Logger @@ -38,7 +38,7 @@ class ElasticRequestLogFilter implements Filter { protected FilterConfig filterConfig = null protected ExecutionContextFactoryImpl ecfi = null - private ElasticClientImpl elasticClient = null + private ElasticFacade.ElasticClient elasticClient = null private boolean disabled = false final ConcurrentLinkedQueue requestLogQueue = new ConcurrentLinkedQueue<>() @@ -51,15 +51,11 @@ class ElasticRequestLogFilter implements Filter { ecfi = (ExecutionContextFactoryImpl) filterConfig.servletContext.getAttribute("executionContextFactory") if (ecfi == null) ecfi = (ExecutionContextFactoryImpl) Moqui.executionContextFactory - elasticClient = (ElasticClientImpl) (ecfi.elasticFacade.getClient("logger") ?: ecfi.elasticFacade.getDefault()) + elasticClient = (ecfi.elasticFacade.getClient("logger") ?: ecfi.elasticFacade.getDefault()) if (elasticClient == null) { logger.error("In ElasticRequestLogFilter init could not find ElasticClient with name logger or default, not starting") return } - if (elasticClient.esVersionUnder7) { - logger.warn("ElasticClient ${elasticClient.clusterName} has version under 7.0, not starting ElasticRequestLogFilter") - return - } // check for index exists, create with mapping for log doc if not try { diff --git a/framework/src/main/resources/MoquiDefaultConf.xml b/framework/src/main/resources/MoquiDefaultConf.xml index d8dc14c59..eb78c0c52 100644 --- a/framework/src/main/resources/MoquiDefaultConf.xml +++ b/framework/src/main/resources/MoquiDefaultConf.xml @@ -420,8 +420,13 @@ + + - + @@ -523,6 +531,7 @@ + diff --git a/framework/src/test/groovy/PostgresElasticClientTests.groovy b/framework/src/test/groovy/PostgresElasticClientTests.groovy new file mode 100644 index 000000000..bad902fab --- /dev/null +++ b/framework/src/test/groovy/PostgresElasticClientTests.groovy @@ -0,0 +1,701 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * Integration tests for PostgresElasticClient and PostgresSearchLogger. + * + * Requires a running PostgreSQL database configured in Moqui (transactional datasource). + * Bootstraps a Moqui EC directly — no web server needed. + * + * Run with: ./gradlew :framework:test --tests PostgresElasticClientTests + */ + +import org.moqui.Moqui +import org.moqui.context.ExecutionContext +import org.moqui.context.ElasticFacade +import org.moqui.impl.context.PostgresElasticClient +import org.moqui.impl.context.ExecutionContextFactoryImpl +import org.moqui.util.MNode +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions + +/** + * Integration tests for the PostgreSQL-backed ElasticClient. + * + * Spins up a real PostgresElasticClient against the configured datasource and exercises + * all major operations: schema init, createIndex, index, get, search, update, delete, + * bulk operations, and query translation. + */ +class PostgresElasticClientTests { + + static ExecutionContext ec + static PostgresElasticClient pgClient + static final String TEST_INDEX = "pg_test_documents" + static final String TEST_PREFIX = "_test_pg_" + + @BeforeAll + static void startMoqui() { + // Init Moqui pointing at the test database + ec = Moqui.getExecutionContext() + ExecutionContextFactoryImpl ecfi = (ExecutionContextFactoryImpl) ec.factory + + // Create a test PostgresElasticClient directly via a manually-built MNode + MNode clusterNode = new MNode("cluster", [name: "test-pg", type: "postgres", + url: "transactional", "index-prefix": TEST_PREFIX]) + pgClient = new PostgresElasticClient(clusterNode, ecfi) + } + + @AfterAll + static void stopMoqui() { + // Clean up test data + try { + if (pgClient != null) { + [ + "pg_test_documents_create", "pg_test_documents_with_alias", + "pg_test_documents_delete_test", "pg_test_documents_put_mapping", + "pg_test_documents_crud", "pg_test_documents_get_source", + "pg_test_documents_multi_get", "pg_test_documents_get_null", + "pg_test_documents_update", "pg_test_documents_doc_delete", + "pg_test_documents_bulkindex", "pg_test_documents_bulk_actions", + "pg_test_documents_search_all", "pg_test_documents_search_term", + "pg_test_documents_search_terms", "pg_test_documents_search_fts", + "pg_test_documents_search_bool", "pg_test_documents_search_page", + "pg_test_documents_searchhits", "pg_test_documents_count", + "pg_test_documents_countresp", "pg_test_documents_dbq", + "test_data_doc" + ].each { idx -> + try { pgClient.deleteIndex(idx) } catch (Throwable ignored) {} + } + } + } catch (Throwable ignored) {} + try { if (ec != null) ec.destroy() } catch (Throwable ignored) {} + } + + @BeforeEach + void beginTx() { + ec.transaction.begin(null) + ec.artifactExecution.disableAuthz() + } + + @AfterEach + void commitTx() { + ec.artifactExecution.enableAuthz() + if (ec.transaction.isTransactionInPlace()) ec.transaction.commit() + } + + // ============================ + // clusterName / location + // ============================ + + @Test + @DisplayName("clusterName returns configured name") + void clusterName_returnsConfiguredName() { + Assertions.assertEquals("test-pg", pgClient.clusterName) + } + + @Test + @DisplayName("clusterLocation contains postgres keyword") + void clusterLocation_containsPostgresKeyword() { + Assertions.assertTrue(pgClient.clusterLocation.contains("postgres")) + } + + @Test + @DisplayName("getServerInfo returns postgres version map") + void getServerInfo_returnsPostgresVersionMap() { + Map info = pgClient.getServerInfo() + Assertions.assertNotNull(info) + Assertions.assertEquals("postgres", info.get("cluster_name")) + Object version = info.get("version") + Assertions.assertNotNull(version) + } + + // ============================ + // Index management + // ============================ + + @Test + @DisplayName("createIndex and indexExists") + void createIndex_andIndexExists() { + String idx = TEST_INDEX + "_create" + try { + pgClient.createIndex(idx, [properties: [name: [type: "text"]]], null) + Assertions.assertTrue(pgClient.indexExists(idx), "index should exist after createIndex") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("indexExists returns false for non-existent index") + void indexExists_returnsFalseForNonExistent() { + Assertions.assertFalse(pgClient.indexExists("pg_test_does_not_exist_xyz")) + } + + @Test + @DisplayName("createIndex with alias and aliasExists") + void createIndex_withAlias_aliasExists() { + String idx = TEST_INDEX + "_with_alias" + String alias = idx + "_alias" + try { + pgClient.createIndex(idx, null, alias) + Assertions.assertTrue(pgClient.indexExists(idx), "index should exist") + Assertions.assertTrue(pgClient.aliasExists(alias), "alias should exist") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("deleteIndex removes documents and metadata") + void deleteIndex_removesDocumentsAndMetadata() { + String idx = TEST_INDEX + "_delete_test" + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "doc001", [name: "To Be Deleted"]) + pgClient.deleteIndex(idx) + Assertions.assertFalse(pgClient.indexExists(idx), "index should not exist after delete") + Assertions.assertNull(pgClient.get(idx, "doc001"), "document should not exist after index delete") + } + + @Test + @DisplayName("putMapping updates mapping on existing index") + void putMapping_updatesMappingOnExistingIndex() { + String idx = TEST_INDEX + "_put_mapping" + try { + pgClient.createIndex(idx, [properties: [name: [type: "text"]]], null) + // putMapping should succeed without throwing + pgClient.putMapping(idx, [properties: [name: [type: "text"], email: [type: "keyword"]]]) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Document CRUD + // ============================ + + @Test + @DisplayName("index and get roundtrip") + void index_andGetRoundtrip() { + String idx = TEST_INDEX + "_crud" + try { + pgClient.createIndex(idx, null, null) + Map doc = [name: "Alice", email: "alice@example.com", age: 30] + pgClient.index(idx, "alice001", doc) + Map retrieved = pgClient.get(idx, "alice001") + Assertions.assertNotNull(retrieved, "document should be retrievable") + Map source = (Map) retrieved.get("_source") + Assertions.assertNotNull(source) + Assertions.assertEquals("Alice", source.get("name")) + Assertions.assertEquals("alice@example.com", source.get("email")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("getSource returns only source map") + void getSource_returnsOnlySourceMap() { + String idx = TEST_INDEX + "_get_source" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "bob001", [name: "Bob", role: "admin"]) + Map source = pgClient.getSource(idx, "bob001") + Assertions.assertNotNull(source) + Assertions.assertEquals("Bob", source.get("name")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("get with _idList returns multiple docs") + void get_withIdList_returnsMultipleDocs() { + String idx = TEST_INDEX + "_multi_get" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "m1", [name: "Multi1"]) + pgClient.index(idx, "m2", [name: "Multi2"]) + pgClient.index(idx, "m3", [name: "Multi3"]) + List docs = pgClient.get(idx, ["m1", "m3"]) + Assertions.assertEquals(2, docs.size()) + Set ids = docs.collect { (String) it.get("_id") }.toSet() + Assertions.assertTrue(ids.contains("m1")) + Assertions.assertTrue(ids.contains("m3")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("get returns null for non-existent document") + void get_returnsNullForNonExistent() { + String idx = TEST_INDEX + "_get_null" + try { + pgClient.createIndex(idx, null, null) + Map result = pgClient.get(idx, "doesnotexist") + Assertions.assertNull(result) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("update merges fields") + void update_mergesFields() { + String idx = TEST_INDEX + "_update" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "upd001", [name: "Carol", status: "active"]) + pgClient.update(idx, "upd001", [status: "inactive", rating: 5]) + Map source = pgClient.getSource(idx, "upd001") + Assertions.assertNotNull(source) + // original name field should still be there (merge) + Assertions.assertEquals("Carol", source.get("name")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("delete removes document") + void delete_removesDocument() { + String idx = TEST_INDEX + "_doc_delete" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "del001", [name: "To Delete"]) + Assertions.assertNotNull(pgClient.get(idx, "del001"), "doc should exist before delete") + pgClient.delete(idx, "del001") + Assertions.assertNull(pgClient.get(idx, "del001"), "doc should not exist after delete") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Bulk operations + // ============================ + + @Test + @DisplayName("bulkIndex inserts multiple documents") + void bulkIndex_insertsMultipleDocuments() { + String idx = TEST_INDEX + "_bulkindex" + try { + pgClient.createIndex(idx, null, null) + List docs = (1..20).collect { i -> + [productId: "PROD_${String.format('%03d', i)}", name: "Product ${i}", + category: i % 2 == 0 ? "category_a" : "category_b", price: i * 10.0] + } + pgClient.bulkIndex(idx, "productId", docs) + + // Verify a sampling + Map p1source = pgClient.getSource(idx, "PROD_001") + Assertions.assertNotNull(p1source, "PROD_001 should exist after bulkIndex") + Assertions.assertEquals("Product 1", p1source.get("name")) + + Map p10source = pgClient.getSource(idx, "PROD_010") + Assertions.assertNotNull(p10source, "PROD_010 should exist") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("bulk with index and delete actions") + void bulk_withIndexAndDeleteActions() { + String idx = TEST_INDEX + "_bulk_actions" + try { + pgClient.createIndex(idx, null, null) + // First insert two docs + pgClient.index(idx, "ba001", [name: "To Keep"]) + pgClient.index(idx, "ba002", [name: "To Delete"]) + + // Bulk: index new + delete existing + List actions = [ + [index: [_index: "${TEST_PREFIX}${idx}", _id: "ba003"]], + [name: "New Doc"], + [delete: [_index: "${TEST_PREFIX}${idx}", _id: "ba002"]], + [:] // no source for delete + ] + pgClient.bulk(idx, actions) + + Assertions.assertNotNull(pgClient.get(idx, "ba001"), "ba001 should still exist") + // ba002 delete and ba003 index via raw bulk - verify ba001 still there + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Search + // ============================ + + @Test + @DisplayName("search - match_all returns all documents") + void search_matchAllReturnsAll() { + String idx = TEST_INDEX + "_search_all" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "s1", [name: "Alpha Widget"]) + pgClient.index(idx, "s2", [name: "Beta Gadget"]) + pgClient.index(idx, "s3", [name: "Gamma Tool"]) + + Map result = pgClient.search(idx, [query: [match_all: [:]], size: 20]) + Assertions.assertNotNull(result) + Map hits = (Map) result.get("hits") + Assertions.assertNotNull(hits) + Map total = (Map) hits.get("total") + Assertions.assertNotNull(total) + long totalValue = ((Number) total.get("value")).longValue() + Assertions.assertEquals(3L, totalValue, "should return 3 documents") + + List hitList = (List) hits.get("hits") + Assertions.assertEquals(3, hitList.size()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - term query filters documents") + void search_termQueryFilters() { + String idx = TEST_INDEX + "_search_term" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "t1", [status: "ACTIVE", name: "Active Widget"]) + pgClient.index(idx, "t2", [status: "INACTIVE", name: "Inactive Gadget"]) + pgClient.index(idx, "t3", [status: "ACTIVE", name: "Active Tool"]) + + Map result = pgClient.search(idx, [ + query: [term: [status: "ACTIVE"]], + size: 20, + track_total_hits: true + ]) + Map hits = (Map) result.get("hits") + Map total = (Map) hits.get("total") + long totalValue = ((Number) total.get("value")).longValue() + Assertions.assertEquals(2L, totalValue, "only ACTIVE docs should be returned") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - terms query with IN") + void search_termsQueryWithIn() { + String idx = TEST_INDEX + "_search_terms" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "tm1", [cat: "a"]) + pgClient.index(idx, "tm2", [cat: "b"]) + pgClient.index(idx, "tm3", [cat: "c"]) + + Map result = pgClient.search(idx, [ + query: [terms: [cat: ["a", "c"]]], + size: 10, + track_total_hits: true + ]) + Map total = (Map) ((Map) result.get("hits")).get("total") + Assertions.assertEquals(2L, ((Number) total.get("value")).longValue()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - full-text query_string") + void search_fullTextQueryString() { + String idx = TEST_INDEX + "_search_fts" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "fts1", [description: "The quick brown fox jumps over the lazy dog"]) + pgClient.index(idx, "fts2", [description: "A completely unrelated document about databases"]) + pgClient.index(idx, "fts3", [description: "Another fox story about running quickly"]) + + Map result = pgClient.search(idx, [ + query: [query_string: [query: "fox", lenient: true]], + size: 10, + track_total_hits: true + ]) + Map total = (Map) ((Map) result.get("hits")).get("total") + long count = ((Number) total.get("value")).longValue() + Assertions.assertTrue(count >= 2, "should find at least 2 fox documents, found ${count}") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - bool must and must_not") + void search_boolMustAndMustNot() { + String idx = TEST_INDEX + "_search_bool" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "b1", [type: "order", status: "placed"]) + pgClient.index(idx, "b2", [type: "order", status: "cancelled"]) + pgClient.index(idx, "b3", [type: "invoice", status: "placed"]) + + Map result = pgClient.search(idx, [ + query: [bool: [ + must: [[term: [type: "order"]]], + must_not: [[term: [status: "cancelled"]]] + ]], + size: 10, + track_total_hits: true + ]) + Map total = (Map) ((Map) result.get("hits")).get("total") + long count = ((Number) total.get("value")).longValue() + Assertions.assertEquals(1L, count, "only non-cancelled orders should match") + List hitList = (List) ((Map) result.get("hits")).get("hits") + Assertions.assertEquals("b1", hitList[0].get("_id")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - pagination from and size") + void search_paginationFromAndSize() { + String idx = TEST_INDEX + "_search_page" + try { + pgClient.createIndex(idx, null, null) + (1..10).each { i -> + pgClient.index(idx, "p${i}", [name: "Doc ${i}", seq: i]) + } + + // Get page 2 (items 5-9 of 10, 5 per page) + Map result = pgClient.search(idx, [ + query: [match_all: [:]], + from: 5, size: 5, + track_total_hits: true + ]) + Map hits = (Map) result.get("hits") + long totalValue = ((Number) ((Map) hits.get("total")).get("value")).longValue() + List hitList = (List) hits.get("hits") + Assertions.assertEquals(10L, totalValue, "total should reflect all 10 docs") + Assertions.assertEquals(5, hitList.size(), "page size should be 5") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("searchHits returns list directly") + void searchHits_returnsListDirectly() { + String idx = TEST_INDEX + "_searchhits" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "sh1", [x: 1]) + pgClient.index(idx, "sh2", [x: 2]) + List hits = pgClient.searchHits(idx, [query: [match_all: [:]], size: 10]) + Assertions.assertNotNull(hits) + Assertions.assertEquals(2, hits.size()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Count + // ============================ + + @Test + @DisplayName("count returns correct total") + void count_returnsCorrectTotal() { + String idx = TEST_INDEX + "_count" + try { + pgClient.createIndex(idx, null, null) + (1..7).each { i -> pgClient.index(idx, "c${i}", [seq: i]) } + + long cnt = pgClient.count(idx, [query: [match_all: [:]]]) + Assertions.assertEquals(7L, cnt) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("countResponse returns map with count key") + void countResponse_returnsMapWithCountKey() { + String idx = TEST_INDEX + "_countresp" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "cr1", [v: 1]) + pgClient.index(idx, "cr2", [v: 2]) + Map resp = pgClient.countResponse(idx, [query: [match_all: [:]]]) + Assertions.assertNotNull(resp) + Assertions.assertTrue(resp.containsKey("count")) + Assertions.assertEquals(2L, ((Number) resp.get("count")).longValue()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // deleteByQuery + // ============================ + + @Test + @DisplayName("deleteByQuery removes matching documents") + void deleteByQuery_removesMatchingDocuments() { + String idx = TEST_INDEX + "_dbq" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "dbq1", [status: "STALE"]) + pgClient.index(idx, "dbq2", [status: "STALE"]) + pgClient.index(idx, "dbq3", [status: "KEEP"]) + + Integer deleted = pgClient.deleteByQuery(idx, [term: [status: "STALE"]]) + Assertions.assertNotNull(deleted) + Assertions.assertEquals(2, deleted.intValue()) + Assertions.assertNull(pgClient.get(idx, "dbq1"), "STALE doc should be deleted") + Assertions.assertNotNull(pgClient.get(idx, "dbq3"), "KEEP doc should remain") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // PIT (stateless for postgres) + // ============================ + + @Test + @DisplayName("getPitId returns non-null token") + void getPitId_returnsNonNullToken() { + String pit = pgClient.getPitId(TEST_INDEX, "1m") + Assertions.assertNotNull(pit) + Assertions.assertTrue(pit.startsWith("pg::")) + } + + @Test + @DisplayName("deletePit is a no-op") + void deletePit_isNoOp() { + // Should not throw + pgClient.deletePit("pg::test::12345") + } + + // ============================ + // validateQuery + // ============================ + + @Test + @DisplayName("validateQuery returns null for valid query") + void validateQuery_returnsNullForValidQuery() { + Map result = pgClient.validateQuery(TEST_INDEX, [term: [status: "active"]], false) + Assertions.assertNull(result, "valid query should return null") + } + + // ============================ + // bulkIndexDataDocument + // ============================ + + @Test + @DisplayName("bulkIndexDataDocument strips metadata and indexes documents") + void bulkIndexDataDocument_stripsMetadataAndIndexes() { + List docs = [ + [_index: "test", _type: "TestDataDoc", _id: "tdd001", + _timestamp: System.currentTimeMillis(), productId: "P001", name: "Test Product One"], + [_index: "test", _type: "TestDataDoc", _id: "tdd002", + _timestamp: System.currentTimeMillis(), productId: "P002", name: "Test Product Two"], + ] + try { + pgClient.bulkIndexDataDocument(docs) + + // Documents should be in 'test_data_doc' index (ddIdToEsIndex("TestDataDoc")) + String expectedIdx = "test_data_doc" + Map source1 = pgClient.getSource(expectedIdx, "tdd001") + Assertions.assertNotNull(source1, "tdd001 should be indexed") + Assertions.assertEquals("Test Product One", source1.get("name")) + // Metadata should be stripped + Assertions.assertFalse(source1.containsKey("_index"), "_index should be stripped") + Assertions.assertFalse(source1.containsKey("_type"), "_type should be stripped") + Assertions.assertFalse(source1.containsKey("_id"), "_id should be stripped") + } finally { + // cleanup + try { pgClient.deleteIndex("test_data_doc") } catch (Throwable ignored) {} + } + } + + // ============================ + // extractContentText + // ============================ + + @Test + @DisplayName("extractContentText collects all string values") + void extractContentText_collectsAllStringValues() { + Map doc = [ + name: "John Doe", + status: "active", + address: [city: "Atlanta", state: "GA"], + tags: ["enterprise", "premium"], + amount: 299.99 + ] + String text = PostgresElasticClient.extractContentText(doc) + Assertions.assertNotNull(text) + Assertions.assertTrue(text.contains("John Doe"), "should include name") + Assertions.assertTrue(text.contains("active"), "should include status") + Assertions.assertTrue(text.contains("Atlanta"), "should include nested city") + Assertions.assertTrue(text.contains("enterprise"), "should include list items") + } + + @Test + @DisplayName("extractContentText on empty map returns empty string") + void extractContentText_emptyMapReturnsEmpty() { + String text = PostgresElasticClient.extractContentText([:]) + Assertions.assertEquals("", text) + } + + @Test + @DisplayName("extractContentText on null returns empty string") + void extractContentText_nullReturnsEmpty() { + String text = PostgresElasticClient.extractContentText(null) + Assertions.assertEquals("", text) + } + + // ============================ + // JSON serialization + // ============================ + + @Test + @DisplayName("objectToJson and jsonToObject roundtrip") + void objectToJson_andJsonToObject_roundtrip() { + Map original = [name: "Test", values: [1, 2, 3], nested: [key: "value"]] + String json = pgClient.objectToJson(original) + Assertions.assertNotNull(json) + Map roundtripped = (Map) pgClient.jsonToObject(json) + Assertions.assertEquals("Test", roundtripped.get("name")) + Assertions.assertEquals([1, 2, 3], roundtripped.get("values") as List) + } + + // ============================ + // Unsupported REST operations throw + // ============================ + + @Test + @DisplayName("call throws UnsupportedOperationException") + void call_throwsUnsupportedOperation() { + Assertions.assertThrows(UnsupportedOperationException.class, { + pgClient.call(org.moqui.util.RestClient.Method.GET, TEST_INDEX, "/", null, null) + }) + } + + @Test + @DisplayName("callFuture throws UnsupportedOperationException") + void callFuture_throwsUnsupportedOperation() { + Assertions.assertThrows(UnsupportedOperationException.class, { + pgClient.callFuture(org.moqui.util.RestClient.Method.GET, TEST_INDEX, "/", null, null) + }) + } + + @Test + @DisplayName("makeRestClient throws UnsupportedOperationException") + void makeRestClient_throwsUnsupportedOperation() { + Assertions.assertThrows(UnsupportedOperationException.class, { + pgClient.makeRestClient(org.moqui.util.RestClient.Method.GET, TEST_INDEX, "/", null) + }) + } +} diff --git a/framework/src/test/groovy/PostgresSearchSuite.groovy b/framework/src/test/groovy/PostgresSearchSuite.groovy new file mode 100644 index 000000000..fb42ac1b6 --- /dev/null +++ b/framework/src/test/groovy/PostgresSearchSuite.groovy @@ -0,0 +1,30 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + */ + +import org.junit.jupiter.api.AfterAll +import org.junit.platform.suite.api.SelectClasses +import org.junit.platform.suite.api.Suite +import org.moqui.Moqui + +/** + * JUnit Platform Suite for PostgreSQL search backend tests. + * + * This suite is separate from MoquiSuite because it requires a live PostgreSQL database. + * It will NOT run as part of the main MoquiSuite — it is opt-in via: + * + * ./gradlew :framework:test --tests PostgresSearchSuite + * + * Or run individual test classes: + * ./gradlew :framework:test --tests PostgresSearchTranslatorTests + * ./gradlew :framework:test --tests PostgresElasticClientTests + */ +@Suite +@SelectClasses([PostgresSearchTranslatorTests.class, PostgresElasticClientTests.class]) +class PostgresSearchSuite { + @AfterAll + static void destroyMoqui() { + Moqui.destroyActiveExecutionContextFactory() + } +} diff --git a/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy new file mode 100644 index 000000000..060b66384 --- /dev/null +++ b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy @@ -0,0 +1,478 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * Unit tests for ElasticQueryTranslator — no database connection required. + * Tests that ES Query DSL is correctly translated to PostgreSQL SQL. + */ +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions +import org.moqui.impl.context.ElasticQueryTranslator +import org.moqui.impl.context.ElasticQueryTranslator.TranslatedQuery +import org.moqui.impl.context.ElasticQueryTranslator.QueryResult + +class PostgresSearchTranslatorTests { + + // ============================================================ + // TranslatedQuery / translateSearchMap + // ============================================================ + + @Test + @DisplayName("translateSearchMap - pagination defaults") + void translateSearchMap_paginationDefaults() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([:]) + Assertions.assertEquals(0, tq.fromOffset, "default from should be 0") + Assertions.assertEquals(20, tq.sizeLimit, "default size should be 20") + Assertions.assertEquals("TRUE", tq.whereClause) + } + + @Test + @DisplayName("translateSearchMap - explicit pagination") + void translateSearchMap_explicitPagination() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([from: 40, size: 10]) + Assertions.assertEquals(40, tq.fromOffset) + Assertions.assertEquals(10, tq.sizeLimit) + } + + @Test + @DisplayName("translateSearchMap - query_string sets tsqueryExpr") + void translateSearchMap_queryStringSetstsqueryExpr() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([ + query: [query_string: [query: "hello world", lenient: true]] + ]) + Assertions.assertNotNull(tq.tsqueryExpr, "tsqueryExpr should be set for query_string") + Assertions.assertTrue(tq.whereClause.contains("content_tsv"), "WHERE should use content_tsv") + } + + @Test + @DisplayName("translateSearchMap - sort spec") + void translateSearchMap_sortSpec() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([ + sort: [[postDate: [order: "desc"]]] + ]) + Assertions.assertNotNull(tq.orderBy, "orderBy should be set") + Assertions.assertTrue(tq.orderBy.contains("DESC"), "orderBy should be DESC") + } + + @Test + @DisplayName("translateSearchMap - highlight fields extracted") + void translateSearchMap_highlightFieldsExtracted() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([ + query: [match_all: [:]], + highlight: [fields: [title: [:], description: [:]]] + ]) + Assertions.assertTrue(tq.highlightFields.containsKey("title")) + Assertions.assertTrue(tq.highlightFields.containsKey("description")) + } + + // ============================================================ + // match_all + // ============================================================ + + @Test + @DisplayName("match_all translates to TRUE") + void matchAll_translatesTrue() { + QueryResult qr = ElasticQueryTranslator.translateQuery([match_all: [:]]) + Assertions.assertEquals("TRUE", qr.clause) + Assertions.assertTrue(qr.params.isEmpty()) + } + + // ============================================================ + // query_string + // ============================================================ + + @Test + @DisplayName("query_string translates to websearch_to_tsquery with tsqueryExpr") + void queryString_translatesWebsearchToTsquery() { + QueryResult qr = ElasticQueryTranslator.translateQuery([query_string: [query: "moqui framework"]]) + Assertions.assertTrue(qr.clause.contains("content_tsv"), "clause should use content_tsv") + Assertions.assertTrue(qr.clause.contains("@@"), "clause should have @@ operator") + Assertions.assertNotNull(qr.tsqueryExpr, "should produce tsqueryExpr for scoring") + Assertions.assertFalse(qr.params.isEmpty(), "should have at least one param for the query string") + } + + @Test + @DisplayName("query_string wildcard stripped for tsquery") + void queryString_wildcardStripped() { + QueryResult qr = ElasticQueryTranslator.translateQuery([query_string: [query: "moqui*"]]) + // Wildcard queries are cleaned to simple word for websearch_to_tsquery + Assertions.assertTrue(qr.clause.contains("content_tsv")) + } + + // ============================================================ + // term + // ============================================================ + + @Test + @DisplayName("term translates to field = value") + void term_translatesEquality() { + QueryResult qr = ElasticQueryTranslator.translateQuery([term: [status: "ACTIVE"]]) + Assertions.assertTrue(qr.clause.contains("->>'status'"), "should use ->>'status'") + Assertions.assertTrue(qr.clause.contains("="), "should use equality") + Assertions.assertEquals(1, qr.params.size()) + Assertions.assertEquals("ACTIVE", qr.params[0]) + } + + @Test + @DisplayName("term on _id translates to doc_id equality") + void term_onIdFieldUsesDocId() { + QueryResult qr = ElasticQueryTranslator.translateQuery([term: ["_id": "TEST_001"]]) + Assertions.assertTrue(qr.clause.contains("doc_id"), "should use doc_id for _id field") + Assertions.assertEquals("TEST_001", qr.params[0]) + } + + @Test + @DisplayName("term on nested field path uses JSONB path access") + void term_nestedFieldPath() { + QueryResult qr = ElasticQueryTranslator.translateQuery([term: ["address.city": "Atlanta"]]) + Assertions.assertTrue(qr.clause.contains("document->'address'->>'city'"), "should use nested path") + Assertions.assertEquals("Atlanta", qr.params[0]) + } + + // ============================================================ + // terms + // ============================================================ + + @Test + @DisplayName("terms translates to IN clause") + void terms_translatesInClause() { + QueryResult qr = ElasticQueryTranslator.translateQuery([terms: [statusId: ["ACTIVE", "PENDING", "DRAFT"]]]) + Assertions.assertTrue(qr.clause.contains("IN"), "should use IN operator") + Assertions.assertEquals(3, qr.params.size()) + Assertions.assertTrue(qr.params.containsAll(["ACTIVE", "PENDING", "DRAFT"])) + } + + @Test + @DisplayName("terms with empty list translates to FALSE") + void terms_emptyListTranslatesFalse() { + QueryResult qr = ElasticQueryTranslator.translateQuery([terms: [statusId: []]]) + Assertions.assertEquals("FALSE", qr.clause) + } + + // ============================================================ + // range + // ============================================================ + + @Test + @DisplayName("range with gte and lte") + void range_gteAndLte() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [orderDate: [gte: "2024-01-01", lte: "2024-12-31"]]]) + Assertions.assertTrue(qr.clause.contains(">="), "should have >=") + Assertions.assertTrue(qr.clause.contains("<="), "should have <=") + Assertions.assertEquals(2, qr.params.size()) + } + + @Test + @DisplayName("range with gt only") + void range_gtOnly() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [amount: [gt: "100"]]]) + Assertions.assertTrue(qr.clause.contains(">"), "should have >") + Assertions.assertFalse(qr.clause.contains(">="), "should not have >= if only gt") + Assertions.assertEquals(1, qr.params.size()) + Assertions.assertEquals("100", qr.params[0]) + } + + @Test + @DisplayName("range on date field gets timestamptz cast") + void range_dateFieldGetsTimestamptzCast() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [orderDate: [gte: "2024-01-01"]]]) + Assertions.assertTrue(qr.clause.contains("::timestamptz"), "date fields should cast to timestamptz") + } + + @Test + @DisplayName("range on amount field gets numeric cast") + void range_amountFieldGetsNumericCast() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [grandTotal: [gte: "0"]]]) + Assertions.assertTrue(qr.clause.contains("::numeric"), "amount fields should cast to numeric") + } + + // ============================================================ + // exists + // ============================================================ + + @Test + @DisplayName("exists translates to JSONB document ? field") + void exists_translatesJsonbHasKey() { + QueryResult qr = ElasticQueryTranslator.translateQuery([exists: [field: "email"]]) + Assertions.assertTrue(qr.clause.contains("document ?"), "should use JSONB ? operator") + Assertions.assertTrue(qr.clause.contains("email")) + } + + // ============================================================ + // bool + // ============================================================ + + @Test + @DisplayName("bool must translates to AND") + void boolMust_translatesAnd() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [must: [ + [term: [type: "ORDER"]], + [term: [status: "PLACED"]] + ]]]) + Assertions.assertTrue(qr.clause.contains("AND"), "must should generate AND") + Assertions.assertEquals(2, qr.params.size()) + } + + @Test + @DisplayName("bool should translates to OR") + void boolShould_translatesOr() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [should: [ + [term: [status: "PLACED"]], + [term: [status: "SHIPPED"]] + ]]]) + Assertions.assertTrue(qr.clause.contains("OR"), "should should generate OR") + } + + @Test + @DisplayName("bool must_not translates to NOT") + void boolMustNot_translatesNot() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [must_not: [ + [term: [status: "CANCELLED"]] + ]]]) + Assertions.assertTrue(qr.clause.toUpperCase().contains("NOT"), "must_not should generate NOT") + } + + @Test + @DisplayName("bool filter translates same as must") + void boolFilter_translatesSameAsMust() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [filter: [ + [term: [tenantId: "DEMO"]] + ]]]) + Assertions.assertTrue(qr.clause.contains("->>'tenantId'"), "filter should translate term like must") + } + + @Test + @DisplayName("bool combined must and must_not") + void boolCombinedMustAndMustNot() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [ + must: [[term: [type: "ORDER"]]], + must_not: [[term: [status: "CANCELLED"]]] + ]]) + Assertions.assertTrue(qr.clause.contains("AND"), "should have AND") + Assertions.assertTrue(qr.clause.toUpperCase().contains("NOT"), "should have NOT") + Assertions.assertEquals(2, qr.params.size()) + } + + // ============================================================ + // nested + // ============================================================ + + @Test + @DisplayName("nested query translates to EXISTS subquery with jsonb_array_elements") + void nested_translatesExistsSubquery() { + QueryResult qr = ElasticQueryTranslator.translateQuery([nested: [ + path: "orderItems", + query: [term: ["orderItems.productId": "PROD_001"]] + ]]) + Assertions.assertTrue(qr.clause.contains("EXISTS"), "nested should use EXISTS subquery") + Assertions.assertTrue(qr.clause.contains("jsonb_array_elements"), "nested should use jsonb_array_elements") + Assertions.assertTrue(qr.clause.contains("orderItems"), "should reference the nested path") + } + + // ============================================================ + // ids + // ============================================================ + + @Test + @DisplayName("ids query translates to doc_id IN list") + void ids_translatesDocIdIn() { + QueryResult qr = ElasticQueryTranslator.translateQuery([ids: [values: ["ID1", "ID2", "ID3"]]]) + Assertions.assertTrue(qr.clause.contains("doc_id IN"), "ids should use doc_id IN") + Assertions.assertEquals(3, qr.params.size()) + } + + @Test + @DisplayName("ids with empty values translates to FALSE") + void ids_emptyValuesTranslatesFalse() { + QueryResult qr = ElasticQueryTranslator.translateQuery([ids: [values: []]]) + Assertions.assertEquals("FALSE", qr.clause) + } + + // ============================================================ + // translateSort + // ============================================================ + + @Test + @DisplayName("translateSort - map with order desc") + void translateSort_mapWithOrderDesc() { + String result = ElasticQueryTranslator.translateSort([[orderDate: [order: "desc"]]]) + Assertions.assertNotNull(result) + Assertions.assertTrue(result.contains("DESC")) + Assertions.assertTrue(result.contains("orderDate")) + } + + @Test + @DisplayName("translateSort - score special field") + void translateSort_scoreSpecialField() { + String result = ElasticQueryTranslator.translateSort([[_score: [order: "desc"]]]) + Assertions.assertNotNull(result) + Assertions.assertTrue(result.contains("_score"), "should produce _score sort entry") + } + + @Test + @DisplayName("translateSort - keyword suffix stripped") + void translateSort_keywordSuffixStripped() { + String result = ElasticQueryTranslator.translateSort([["statusId.keyword": [order: "asc"]]]) + Assertions.assertFalse(result.contains(".keyword"), ".keyword suffix should be stripped") + Assertions.assertTrue(result.contains("statusId")) + } + + @Test + @DisplayName("translateSort - string shorthand") + void translateSort_stringShorthand() { + String result = ElasticQueryTranslator.translateSort(["orderDate"]) + Assertions.assertNotNull(result) + Assertions.assertTrue(result.contains("orderDate")) + } + + @Test + @DisplayName("translateSort - null returns null") + void translateSort_nullReturnsNull() { + String result = ElasticQueryTranslator.translateSort(null) + Assertions.assertNull(result) + } + + @Test + @DisplayName("translateSort - empty list returns null") + void translateSort_emptyListReturnsNull() { + String result = ElasticQueryTranslator.translateSort([]) + Assertions.assertNull(result) + } + + // ============================================================ + // Security: sanitizeFieldName — SQL injection prevention + // ============================================================ + + @Test + @DisplayName("sanitizeFieldName rejects SQL injection via single quote") + void sanitizeFieldName_rejectsSingleQuote() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("status'; DROP TABLE users;--") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects SQL injection via semicolon") + void sanitizeFieldName_rejectsSemicolon() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field;DELETE FROM moqui_search_index") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects parentheses") + void sanitizeFieldName_rejectsParentheses() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field()OR 1=1") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects double dash comment") + void sanitizeFieldName_rejectsDoubleDash() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field--comment") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects null field") + void sanitizeFieldName_rejectsNull() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName(null) + } + } + + @Test + @DisplayName("sanitizeFieldName rejects empty field") + void sanitizeFieldName_rejectsEmpty() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects spaces") + void sanitizeFieldName_rejectsSpaces() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field name") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects UNION SELECT injection") + void sanitizeFieldName_rejectsUnionSelect() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("x' UNION SELECT * FROM pg_shadow--") + } + } + + @Test + @DisplayName("sanitizeFieldName accepts valid field names") + void sanitizeFieldName_acceptsValidNames() { + // These should NOT throw + Assertions.assertEquals("statusId", ElasticQueryTranslator.sanitizeFieldName("statusId")) + Assertions.assertEquals("order.items.quantity", ElasticQueryTranslator.sanitizeFieldName("order.items.quantity")) + Assertions.assertEquals("@timestamp", ElasticQueryTranslator.sanitizeFieldName("@timestamp")) + Assertions.assertEquals("field-name", ElasticQueryTranslator.sanitizeFieldName("field-name")) + Assertions.assertEquals("_id", ElasticQueryTranslator.sanitizeFieldName("_id")) + } + + @Test + @DisplayName("sanitizeFieldName rejects oversized field name") + void sanitizeFieldName_rejectsOversized() { + String longField = "a" * 257 + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName(longField) + } + } + + @Test + @DisplayName("term query with SQL injection field name is rejected") + void term_sqlInjectionFieldRejected() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.translateQuery([term: ["status'; DROP TABLE x;--": "active"]]) + } + } + + @Test + @DisplayName("range query with SQL injection field name is rejected") + void range_sqlInjectionFieldRejected() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.translateQuery([range: ["x' OR '1'='1": [gte: "2024-01-01"]]]) + } + } + + @Test + @DisplayName("exists query with SQL injection field name is rejected") + void exists_sqlInjectionFieldRejected() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.translateQuery([exists: [field: "x'; DELETE FROM moqui_search_index;--"]]) + } + } + + // ============================================================ + // Full searchMap round-trip + // ============================================================ + + @Test + @DisplayName("full searchMap with bool query and sort") + void fullSearchMap_boolQueryAndSort() { + Map searchMap = [ + from: 0, size: 25, + sort: [[orderDate: [order: "desc"]]], + query: [bool: [ + must: [[term: [statusId: "OrderPlaced"]]], + filter: [[range: [orderDate: [gte: "2024-01-01"]]]] + ]], + highlight: [fields: [productName: [:]]] + ] + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap) + Assertions.assertEquals(0, tq.fromOffset) + Assertions.assertEquals(25, tq.sizeLimit) + Assertions.assertNotNull(tq.orderBy) + Assertions.assertTrue(tq.whereClause.contains("AND")) + Assertions.assertTrue(tq.highlightFields.containsKey("productName")) + } +} diff --git a/framework/xsd/moqui-conf-3.xsd b/framework/xsd/moqui-conf-3.xsd index d060a6176..e9138b70d 100644 --- a/framework/xsd/moqui-conf-3.xsd +++ b/framework/xsd/moqui-conf-3.xsd @@ -594,16 +594,29 @@ along with this software (see the LICENSE.md file). If not, see - + + Backend type for this cluster. Use 'elastic' (default) for ElasticSearch/OpenSearch via HTTP REST. + Use 'postgres' to store and search documents in PostgreSQL using JSONB and tsvector. + When type is 'postgres', url should be the entity datasource group name (e.g. 'transactional'). + + + + + + + + For type=elastic: full URL to ElasticSearch/OpenSearch (e.g. http://localhost:9200). + For type=postgres: the Moqui entity datasource group name to use (e.g. 'transactional'), or omit to use the default transactional group. + - Prefix added to all ES index names just before requests - Must follow ES index name requirements (lower case, etc) - No separator character is added, recommend ending with underscore (_) + Prefix added to all index names just before requests. + For type=elastic: Must follow ES index name requirements (lower case, etc). No separator character is added, recommend ending with underscore (_). + For type=postgres: Used as a prefix to the index_name column value in moqui_document table. - - + For type=elastic only: HTTP connection pool max size. + For type=elastic only: HTTP request queue size. From 84f7c48a2757532ead3f9d6d2d7f62ecc8da07cc Mon Sep 17 00:00:00 2001 From: pandor4u <103976470+pandor4u@users.noreply.github.com> Date: Fri, 10 Apr 2026 18:04:03 -0500 Subject: [PATCH 2/3] Improve PostgreSQL search backend: dedicated table routing, ts_headline, parameterized queries - Route HTTP logs to dedicated moqui_http_log table with typed columns instead of generic moqui_document JSONB storage - Route deleteByQuery to dedicated tables (moqui_http_log, moqui_logs) with proper timestamp range extraction for nightly cleanup jobs - Replace Java regex highlights with PostgreSQL ts_headline() for accurate, index-aware snippet generation - Fix update() to read-merge-extract pattern so content_text stays consistent with extractContentText() used by index() - Add searchHttpLogTable() for searching against dedicated HTTP log table - Improve guessCastType() to inspect actual values (epoch millis, decimals, ISO dates) when field name heuristics are ambiguous - Parameterize exists query (use ?? operator) to prevent SQL injection - Handle indexExists/createIndex/count for dedicated table names --- .../context/ElasticQueryTranslator.groovy | 45 +- .../impl/context/PostgresElasticClient.groovy | 510 ++++++++++++++++-- .../PostgresSearchTranslatorTests.groovy | 5 +- 3 files changed, 495 insertions(+), 65 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy index 887aa1237..db25964b9 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy @@ -370,8 +370,9 @@ class ElasticQueryTranslator { List conditions = [] List params = [] - // Determine cast type based on common field name patterns - String castType = guessCastType(field) + // Determine cast type: first try field name heuristics, then inspect actual values + Object sampleValue = rangeSpecMap.get("gte") ?: rangeSpecMap.get("gt") ?: rangeSpecMap.get("lte") ?: rangeSpecMap.get("lt") + String castType = guessCastType(field, sampleValue) Object gte = rangeSpecMap.get("gte") Object gt = rangeSpecMap.get("gt") @@ -397,13 +398,15 @@ class ElasticQueryTranslator { // Validate field name to prevent SQL injection sanitizeFieldName(field) - // For nested paths, check the nested path exists + // Use parameterized JSONB key existence check if (field.contains(".")) { List parts = field.split("\\.") as List String topLevel = parts[0] - qr.clause = "document ? '${topLevel}'" + qr.clause = "document ?? ?" + qr.params = [topLevel] } else { - qr.clause = "document ? '${field}'" + qr.clause = "document ?? ?" + qr.params = [field] } return qr } @@ -521,13 +524,15 @@ class ElasticQueryTranslator { Map rangeSpecMap = (Map) rangeSpec String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field sanitizeFieldName(localField) - String castType = guessCastType(localField) List conditions = [] List params = [] - Object gte = rangeSpecMap.get("gte"); if (gte != null) { conditions.add("(elem->>'${localField}')${castType} >= ?"); params.add(gte.toString()) } - Object gt = rangeSpecMap.get("gt"); if (gt != null) { conditions.add("(elem->>'${localField}')${castType} > ?"); params.add(gt.toString()) } - Object lte = rangeSpecMap.get("lte"); if (lte != null) { conditions.add("(elem->>'${localField}')${castType} <= ?"); params.add(lte.toString()) } - Object lt = rangeSpecMap.get("lt"); if (lt != null) { conditions.add("(elem->>'${localField}')${castType} < ?"); params.add(lt.toString()) } + Object gte = rangeSpecMap.get("gte"); Object gt = rangeSpecMap.get("gt") + Object lte = rangeSpecMap.get("lte"); Object lt = rangeSpecMap.get("lt") + String castType = guessCastType(localField, gte ?: gt ?: lte ?: lt) + if (gte != null) { conditions.add("(elem->>'${localField}')${castType} >= ?"); params.add(gte.toString()) } + if (gt != null) { conditions.add("(elem->>'${localField}')${castType} > ?"); params.add(gt.toString()) } + if (lte != null) { conditions.add("(elem->>'${localField}')${castType} <= ?"); params.add(lte.toString()) } + if (lt != null) { conditions.add("(elem->>'${localField}')${castType} < ?"); params.add(lt.toString()) } qr.clause = conditions ? conditions.join(" AND ") : "TRUE" qr.params = params return qr @@ -605,10 +610,12 @@ class ElasticQueryTranslator { } /** - * Guess the appropriate PostgreSQL cast type for a field name to use in range/sort comparisons. + * Guess the appropriate PostgreSQL cast type for a field to use in range/sort comparisons. + * Uses field name heuristics first, then falls back to inspecting the actual value. * Returns empty string if no cast is needed (use text comparison). */ - private static String guessCastType(String field) { + static String guessCastType(String field, Object sampleValue = null) { + // 1. Field name heuristics String lf = field.toLowerCase() if (lf.contains("date") || lf.contains("stamp") || lf.contains("time") || lf == "@timestamp") { return "::timestamptz" @@ -618,6 +625,20 @@ class ElasticQueryTranslator { lf.contains("number") || lf.contains("num") || lf.contains("id") && lf.endsWith("num")) { return "::numeric" } + + // 2. Inspect the actual value if field name is ambiguous + if (sampleValue != null) { + String sv = sampleValue.toString().trim() + // Large number (> 10 digits) that's all digits — likely epoch millis timestamp + if (sv.matches(/^\d{10,}$/)) return "::numeric" + // Decimal number + if (sv.matches(/^-?\d+\.\d+$/)) return "::numeric" + // Integer + if (sv.matches(/^-?\d+$/)) return "::numeric" + // ISO date pattern + if (sv.matches(/^\d{4}-\d{2}-\d{2}.*/)) return "::timestamptz" + } + return "" } diff --git a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy index a28f247f2..11d09bdff 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy @@ -52,9 +52,22 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { private final static Logger logger = LoggerFactory.getLogger(PostgresElasticClient.class) private final static Set DOC_META_KEYS = new HashSet<>(["_index", "_type", "_id", "_timestamp"]) + /** Index names that map to the dedicated moqui_http_log table */ + private static final Set HTTP_LOG_INDEX_NAMES = new HashSet<>(["moqui_http_log"]) + /** Index names that map to the dedicated moqui_logs table */ + private static final Set APP_LOG_INDEX_NAMES = new HashSet<>(["moqui_logs"]) + /** Jackson mapper shared with ElasticFacadeImpl */ static final ObjectMapper jacksonMapper = ElasticFacadeImpl.jacksonMapper + /** SQL for inserting HTTP request logs into the dedicated moqui_http_log table */ + static final String HTTP_LOG_INSERT_SQL = """ + INSERT INTO moqui_http_log (log_timestamp, remote_ip, remote_user, server_ip, content_type, + request_method, request_scheme, request_host, request_path, request_query, http_version, + response_code, time_initial_ms, time_final_ms, bytes_sent, referrer, agent, session_id, visitor_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """.trim() + /** Shared UPSERT SQL for moqui_document — used by upsertDocument(), bulkIndex(), and bulkIndexDataDocument() */ static final String DOCUMENT_UPSERT_SQL = """ INSERT INTO moqui_document (index_name, doc_id, doc_type, document, content_text, content_tsv, updated_stamp) @@ -280,6 +293,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override boolean indexExists(String index) { if (!index) return false + // Dedicated tables always exist (created in initSchema) + if (isHttpLogIndex(index) || isAppLogIndex(index)) return true String prefixed = prefixIndexName(index) Connection conn = getConnection() PreparedStatement ps = conn.prepareStatement( @@ -307,6 +322,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override void createIndex(String index, Map docMapping, String alias) { + // Dedicated tables are created in initSchema — nothing to do + if (isHttpLogIndex(index) || isAppLogIndex(index)) return createIndex(index, null, docMapping, alias, null) } @@ -376,10 +393,29 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { // Document CRUD // ============================================================ + /** Check if the given index name (raw or prefixed) refers to the dedicated HTTP log table */ + private boolean isHttpLogIndex(String index) { + if (!index) return false + String raw = unprefixIndexName(index.trim()) + return HTTP_LOG_INDEX_NAMES.contains(raw) + } + + /** Check if the given index name (raw or prefixed) refers to the dedicated app logs table */ + private boolean isAppLogIndex(String index) { + if (!index) return false + String raw = unprefixIndexName(index.trim()) + return APP_LOG_INDEX_NAMES.contains(raw) + } + @Override void index(String index, String _id, Map document) { if (!index) throw new IllegalArgumentException("Index name required for index()") if (!_id) throw new IllegalArgumentException("_id required for index()") + // Route HTTP log documents to dedicated table + if (isHttpLogIndex(index)) { + insertHttpLog(document) + return + } String prefixedIndex = prefixIndexName(index) String docJson = objectToJson(document) String contentText = extractContentText(document) @@ -392,31 +428,48 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { if (!_id) throw new IllegalArgumentException("_id required for update()") String prefixedIndex = prefixIndexName(index) String fragmentJson = objectToJson(documentFragment) + + // Merge the fragment into the existing document, then re-extract content_text using + // the same recursive extractContentText() logic used by index()/upsert to keep FTS consistent Connection conn = getConnection() - PreparedStatement ps = conn.prepareStatement(""" - UPDATE moqui_document - SET document = COALESCE(document, '{}'::jsonb) || ?::jsonb, - content_text = ( - SELECT string_agg(val::text, ' ') - FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) - WHERE jsonb_typeof(COALESCE(document, '{}'::jsonb) || ?::jsonb -> kv.key) IN ('string', 'number') - ), - content_tsv = to_tsvector('english', coalesce(( - SELECT string_agg(val::text, ' ') - FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) - ), '')), - updated_stamp = now() - WHERE index_name = ? AND doc_id = ? - """.trim()) + PreparedStatement getPs = conn.prepareStatement( + "SELECT document FROM moqui_document WHERE index_name = ? AND doc_id = ?") try { - ps.setString(1, fragmentJson) - ps.setString(2, fragmentJson) - ps.setString(3, fragmentJson) - ps.setString(4, fragmentJson) - ps.setString(5, prefixedIndex) - ps.setString(6, _id) - ps.executeUpdate() - } finally { ps.close() } + getPs.setString(1, prefixedIndex) + getPs.setString(2, _id) + ResultSet rs = getPs.executeQuery() + try { + if (rs.next()) { + String existingJson = rs.getString("document") + Map existingDoc = existingJson ? (Map) jsonToObject(existingJson) : [:] + // Deep merge: fragment overrides existing top-level keys + Map mergedDoc = new LinkedHashMap<>(existingDoc) + mergedDoc.putAll(documentFragment) + String mergedJson = objectToJson(mergedDoc) + String contentText = extractContentText(mergedDoc) + // Update with the fully merged document and properly extracted content + PreparedStatement updPs = conn.prepareStatement(""" + UPDATE moqui_document + SET document = ?::jsonb, + content_text = ?, + content_tsv = to_tsvector('english', COALESCE(?, '')), + updated_stamp = now() + WHERE index_name = ? AND doc_id = ? + """.trim()) + try { + updPs.setString(1, mergedJson) + updPs.setString(2, contentText) + updPs.setString(3, contentText) + updPs.setString(4, prefixedIndex) + updPs.setString(5, _id) + updPs.executeUpdate() + } finally { updPs.close() } + } else { + logger.warn("update(): document not found in index '${prefixedIndex}' with id '${_id}', inserting as new") + upsertDocument(prefixedIndex, _id, null, fragmentJson, extractContentText(documentFragment)) + } + } finally { rs.close() } + } finally { getPs.close() } } @Override @@ -438,6 +491,11 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override Integer deleteByQuery(String index, Map queryMap) { if (!index) throw new IllegalArgumentException("Index name required for deleteByQuery()") + + // Route to dedicated tables for logs + if (isHttpLogIndex(index)) return deleteByQueryHttpLog(queryMap) + if (isAppLogIndex(index)) return deleteByQueryAppLog(queryMap) + String prefixedIndex = prefixIndexName(index) ElasticQueryTranslator.QueryResult qr = ElasticQueryTranslator.translateQuery(queryMap ?: [match_all: [:]]) @@ -456,6 +514,120 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { } finally { ps.close() } } + /** Delete from the dedicated moqui_http_log table based on query (supports @timestamp range) */ + private Integer deleteByQueryHttpLog(Map queryMap) { + TimestampRange range = extractTimestampRange(queryMap) + if (range == null) { + // match_all → delete everything + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_http_log") + try { return ps.executeUpdate() } finally { ps.close() } + } + List conditions = [] + List params = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.lt != null) { conditions.add("log_timestamp < ?"); params.add(range.lt) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (range.gt != null) { conditions.add("log_timestamp > ?"); params.add(range.gt) } + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_http_log WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + return ps.executeUpdate() + } finally { ps.close() } + } + + /** Delete from the dedicated moqui_logs table based on query (supports @timestamp range) */ + private Integer deleteByQueryAppLog(Map queryMap) { + TimestampRange range = extractTimestampRange(queryMap) + if (range == null) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_logs") + try { return ps.executeUpdate() } finally { ps.close() } + } + List conditions = [] + List params = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.lt != null) { conditions.add("log_timestamp < ?"); params.add(range.lt) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (range.gt != null) { conditions.add("log_timestamp > ?"); params.add(range.gt) } + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_logs WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + return ps.executeUpdate() + } finally { ps.close() } + } + + /** Simple holder for timestamp range bounds extracted from query DSL */ + private static class TimestampRange { + Timestamp lte, lt, gte, gt + } + + /** + * Extract @timestamp range bounds from an ES query map (as used by the nightly cleanup jobs). + * Handles both epoch millis (Long/String number) and ISO date strings. + * Returns null if no timestamp range is found (e.g. match_all). + */ + private static TimestampRange extractTimestampRange(Map queryMap) { + if (queryMap == null) return null + // Direct range query: {range: {@timestamp: {lte: ...}}} + if (queryMap.containsKey("range")) { + return parseTimestampRangeSpec(queryMap) + } + // Bool query with range in must list + Map boolMap = (Map) queryMap.get("bool") + if (boolMap == null) return null + Object mustVal = boolMap.get("must") + List mustList = [] + if (mustVal instanceof List) mustList = (List) mustVal + else if (mustVal instanceof Map) mustList = [(Map) mustVal] + for (Map clause in mustList) { + if (clause.containsKey("range")) { + return parseTimestampRangeSpec(clause) + } + } + return null + } + + private static TimestampRange parseTimestampRangeSpec(Map rangeClause) { + Map rangeMap = (Map) rangeClause.get("range") + if (rangeMap == null) return null + // Look for @timestamp or any timestamp-like field + Map specMap = null + for (String key in ["@timestamp", "log_timestamp"]) { + if (rangeMap.containsKey(key)) { specMap = (Map) rangeMap.get(key); break } + } + if (specMap == null) { + // Try first key + String firstKey = rangeMap.keySet().isEmpty() ? null : (String) rangeMap.keySet().iterator().next() + if (firstKey) specMap = (Map) rangeMap.get(firstKey) + } + if (specMap == null) return null + TimestampRange range = new TimestampRange() + range.lte = toTimestamp(specMap.get("lte")) + range.lt = toTimestamp(specMap.get("lt")) + range.gte = toTimestamp(specMap.get("gte")) + range.gt = toTimestamp(specMap.get("gt")) + if (range.lte == null && range.lt == null && range.gte == null && range.gt == null) return null + return range + } + + /** Convert a value (epoch millis long, numeric string, or ISO string) to a Timestamp */ + private static Timestamp toTimestamp(Object val) { + if (val == null) return null + if (val instanceof Number) return new Timestamp(((Number) val).longValue()) + String s = val.toString().trim() + if (!s) return null + // Try epoch millis + try { return new Timestamp(Long.parseLong(s)) } catch (NumberFormatException ignored) {} + // Try ISO format + try { return Timestamp.valueOf(s.replace('T', ' ').replace('Z', '')) } catch (Exception ignored) {} + return null + } + @Override void bulk(String index, List actionSourceList) { if (!actionSourceList) return @@ -508,6 +680,11 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override void bulkIndex(String index, String docType, String idField, List documentList, boolean refresh) { if (!documentList) return + // Route HTTP log documents to dedicated table + if (isHttpLogIndex(index)) { + bulkInsertHttpLogs(documentList) + return + } String prefixedIndex = prefixIndexName(index) boolean hasId = idField != null && !idField.isEmpty() @@ -593,9 +770,13 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override Map search(String index, Map searchMap) { - if (index && (index == 'moqui_logs' || prefixIndexName(index) == prefixIndexName('moqui_logs'))) { + // Route dedicated log tables + if (index && isAppLogIndex(index)) { return searchLogsTable(searchMap ?: [:]) } + if (index && isHttpLogIndex(index)) { + return searchHttpLogTable(searchMap ?: [:]) + } ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap ?: [:]) @@ -604,9 +785,6 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { return [hits: [total: [value: 0, relation: "eq"], hits: []]] } - String scoreExpr = tq.tsqueryExpr ? - "ts_rank_cd(content_tsv, ${tq.tsqueryExpr})" : "1.0::float" - String idxPlaceholders = indexNames.collect { "?" }.join(", ") String whereClause = "index_name IN (${idxPlaceholders})" List allParams = new ArrayList<>(indexNames) @@ -621,11 +799,28 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { String orderByClause = tq.orderBy ?: (tq.tsqueryExpr ? "_score DESC" : "updated_stamp DESC") + // Build highlight columns (use PostgreSQL ts_headline when we have a tsquery) + boolean useDbHighlights = tq.highlightFields && tq.tsqueryExpr + List hlColumnExprs = [] + List hlFieldNames = [] + List hlParams = [] + if (useDbHighlights) { + for (String hlField in tq.highlightFields.keySet()) { + String jsonPath = ElasticQueryTranslator.fieldToJsonPath("document", hlField) + String hlExpr = ElasticQueryTranslator.buildHighlightExpr(jsonPath, tq.tsqueryExpr) + hlColumnExprs.add("${hlExpr} AS hl_${hlFieldNames.size()}".toString()) + hlFieldNames.add(hlField) + hlParams.addAll(tq.tsqueryParams) + } + } + + String hlSelect = hlColumnExprs ? ", " + hlColumnExprs.join(", ") : "" + String countSql = "SELECT COUNT(*) FROM moqui_document WHERE ${whereClause}" long totalCount = 0L String mainSql = """ - SELECT doc_id, index_name, doc_type, document, ${buildScoreSelect(tq)} AS _score + SELECT doc_id, index_name, doc_type, document, ${buildScoreSelect(tq)} AS _score${hlSelect} FROM moqui_document WHERE ${whereClause} ORDER BY ${orderByClause} @@ -645,6 +840,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { List mainParams = [] if (tq.tsqueryExpr) mainParams.addAll(tq.tsqueryParams) + // Add highlight params (one set of tsquery params per highlight field) + mainParams.addAll(hlParams) mainParams.addAll(allParams) mainParams.add(tq.sizeLimit) mainParams.add(tq.fromOffset) @@ -666,8 +863,13 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { Map hit = [_index: idxName, _id: docId, _type: docType, _score: score, _source: source] as Map - if (tq.highlightFields && tq.tsqueryExpr) { - Map> highlights = buildHighlights(source, tq) + // Read ts_headline results from result set columns + if (useDbHighlights) { + Map> highlights = [:] + for (int h = 0; h < hlFieldNames.size(); h++) { + String hlResult = rs.getString("hl_${h}") + if (hlResult) highlights.put(hlFieldNames[h], [hlResult]) + } if (highlights) hit.put("highlight", highlights) } @@ -784,6 +986,116 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { } finally { ps.close() } } + /** Search the dedicated moqui_http_log table (mirrors searchLogsTable pattern) */ + private Map searchHttpLogTable(Map searchMap) { + ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap) + + String rawQuery = null + Map queryMap = (Map) searchMap?.get("query") + if (queryMap) { + Map qsMap = (Map) queryMap.get("query_string") + if (qsMap) rawQuery = (String) qsMap.get("query") + } + + List conditions = [] + List params = [] + + if (rawQuery) { + java.util.regex.Matcher m = (rawQuery =~ /@timestamp\s*:\s*\[\s*(\*|\d+)\s+TO\s+(\*|\d+)\s*\]/) + if (m.find()) { + String fromVal = m.group(1) + String toVal = m.group(2) + if (fromVal != '*') { + conditions.add("log_timestamp >= ?") + params.add(new java.sql.Timestamp(Long.parseLong(fromVal))) + } + if (toVal != '*') { + conditions.add("log_timestamp <= ?") + params.add(new java.sql.Timestamp(Long.parseLong(toVal))) + } + } + } + + // Also check for range query in bool.must + if (queryMap) { + TimestampRange range = extractTimestampRange(queryMap) + if (range != null) { + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.lt != null) { conditions.add("log_timestamp < ?"); params.add(range.lt) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (range.gt != null) { conditions.add("log_timestamp > ?"); params.add(range.gt) } + } + } + + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + + Connection conn = getConnection() + long totalCount = 0L + if (tq.trackTotal) { + PreparedStatement countPs = conn.prepareStatement("SELECT COUNT(*) FROM moqui_http_log WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(countPs, i + 1, params[i]) + ResultSet rs = countPs.executeQuery() + try { if (rs.next()) totalCount = rs.getLong(1) } finally { rs.close() } + } finally { countPs.close() } + } + + String mainSql = """ + SELECT log_id, log_timestamp, remote_ip, remote_user, server_ip, content_type, + request_method, request_scheme, request_host, request_path, request_query, + http_version, response_code, time_initial_ms, time_final_ms, bytes_sent, + referrer, agent, session_id, visitor_id + FROM moqui_http_log + WHERE ${whereClause} + ORDER BY log_timestamp DESC + LIMIT ? OFFSET ? + """.trim() + + PreparedStatement ps = conn.prepareStatement(mainSql) + try { + int pIdx = 0 + for (int i = 0; i < params.size(); i++) setParam(ps, ++pIdx, params[i]) + ps.setInt(++pIdx, tq.sizeLimit) + ps.setInt(++pIdx, tq.fromOffset) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + long logId = rs.getLong("log_id") + java.sql.Timestamp ts = rs.getTimestamp("log_timestamp") + Map source = [ + "@timestamp" : ts?.time, + remote_ip : rs.getString("remote_ip"), + remote_user : rs.getString("remote_user"), + server_ip : rs.getString("server_ip"), + content_type : rs.getString("content_type"), + request_method : rs.getString("request_method"), + request_scheme : rs.getString("request_scheme"), + request_host : rs.getString("request_host"), + request_path : rs.getString("request_path"), + request_query : rs.getString("request_query"), + http_version : rs.getString("http_version"), + response : rs.getInt("response_code"), + time_initial_ms : rs.getLong("time_initial_ms"), + time_final_ms : rs.getLong("time_final_ms"), + bytes : rs.getLong("bytes_sent"), + referrer : rs.getString("referrer"), + agent : rs.getString("agent"), + session : rs.getString("session_id"), + visitor_id : rs.getString("visitor_id"), + ] as Map + hits.add([_index: "moqui_http_log", _id: String.valueOf(logId), + _type: "MoquiHttpRequest", _score: 1.0, _source: source] as Map) + } + return [hits: [total: [value: totalCount, relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } catch (Throwable t) { + logger.error("searchHttpLogTable error: " + t.message, t) + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } finally { ps.close() } + } + @Override List searchHits(String index, Map searchMap) { Map result = search(index, searchMap) @@ -802,10 +1114,51 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override long count(String index, Map countMap) { + // Route dedicated tables + if (isHttpLogIndex(index)) return countHttpLog(countMap) + if (isAppLogIndex(index)) return countAppLog(countMap) Map result = countResponse(index, countMap) return ((Number) result.get("count"))?.longValue() ?: 0L } + private long countHttpLog(Map countMap) { + TimestampRange range = countMap?.get("query") ? extractTimestampRange((Map) countMap.get("query")) : null + String sql = "SELECT COUNT(*) FROM moqui_http_log" + List params = [] + if (range != null) { + List conditions = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (conditions) sql += " WHERE " + conditions.join(" AND ") + } + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + ResultSet rs = ps.executeQuery() + try { return rs.next() ? rs.getLong(1) : 0L } finally { rs.close() } + } finally { ps.close() } + } + + private long countAppLog(Map countMap) { + TimestampRange range = countMap?.get("query") ? extractTimestampRange((Map) countMap.get("query")) : null + String sql = "SELECT COUNT(*) FROM moqui_logs" + List params = [] + if (range != null) { + List conditions = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (conditions) sql += " WHERE " + conditions.join(" AND ") + } + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + ResultSet rs = ps.executeQuery() + try { return rs.next() ? rs.getLong(1) : 0L } finally { rs.close() } + } finally { ps.close() } + } + @Override Map countResponse(String index, Map countMap) { if (!countMap) countMap = [query: [match_all: [:]]] @@ -1132,30 +1485,85 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { return "1.0::float" } - private static Map> buildHighlights(Map source, ElasticQueryTranslator.TranslatedQuery tq) { - Map> highlights = [:] - if (!tq.tsqueryExpr || !tq.highlightFields) return highlights - String firstParam = tq.params ? tq.params[0]?.toString() : null - if (!firstParam) return highlights - for (String field in tq.highlightFields.keySet()) { - Object fieldVal = source.get(field) - if (fieldVal instanceof String) { - String text = (String) fieldVal - String highlighted = simpleHighlight(text, firstParam) - if (highlighted != text) highlights.put(field, [highlighted]) + // ============================================================ + // HTTP log insert helpers + // ============================================================ + + /** Insert a single HTTP log record into the dedicated moqui_http_log table */ + private void insertHttpLog(Map document) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(HTTP_LOG_INSERT_SQL) + try { + setHttpLogParams(ps, document) + ps.executeUpdate() + } finally { ps.close() } + } + + /** Bulk insert HTTP log records into the dedicated moqui_http_log table */ + private void bulkInsertHttpLogs(List documentList) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(HTTP_LOG_INSERT_SQL) + try { + int batchSize = 0 + for (Map doc in documentList) { + setHttpLogParams(ps, doc) + ps.addBatch() + batchSize++ + if (batchSize >= 500) { + ps.executeBatch() + batchSize = 0 + } } - } - return highlights + if (batchSize > 0) ps.executeBatch() + } finally { ps.close() } } - private static String simpleHighlight(String text, String query) { - if (!text || !query) return text - List terms = query.replaceAll(/["()+\-]/, ' ').split(/\s+/).findAll { it.length() > 2 } as List - String result = text - for (String term in terms) { - result = result.replaceAll("(?i)\\b${java.util.regex.Pattern.quote(term)}\\b", "\$0") - } - return result + /** Set parameters on an HTTP_LOG_INSERT_SQL PreparedStatement from an HTTP log document map */ + private static void setHttpLogParams(PreparedStatement ps, Map doc) { + // @timestamp: epoch millis long (from ElasticRequestLogFilter) + Object tsObj = doc.get("@timestamp") + if (tsObj instanceof Number) ps.setTimestamp(1, new Timestamp(((Number) tsObj).longValue())) + else if (tsObj != null) ps.setTimestamp(1, new Timestamp(Long.parseLong(tsObj.toString()))) + else ps.setTimestamp(1, new Timestamp(System.currentTimeMillis())) + + setStrParam(ps, 2, doc.get("remote_ip")) + setStrParam(ps, 3, doc.get("remote_user")) + setStrParam(ps, 4, doc.get("server_ip")) + setStrParam(ps, 5, doc.get("content_type")) + setStrParam(ps, 6, doc.get("request_method")) + setStrParam(ps, 7, doc.get("request_scheme")) + setStrParam(ps, 8, doc.get("request_host")) + setStrParam(ps, 9, doc.get("request_path")) + setStrParam(ps, 10, doc.get("request_query")) + + // http_version: stored as text (filter sends a float/half_float) + Object httpVer = doc.get("http_version") + setStrParam(ps, 11, httpVer != null ? httpVer.toString() : null) + + // response code + Object respObj = doc.get("response") + if (respObj instanceof Number) ps.setInt(12, ((Number) respObj).intValue()) + else if (respObj != null) try { ps.setInt(12, Integer.parseInt(respObj.toString())) } catch (Exception e) { ps.setNull(12, Types.INTEGER) } + else ps.setNull(12, Types.INTEGER) + + // timing + setLongParam(ps, 13, doc.get("time_initial_ms")) + setLongParam(ps, 14, doc.get("time_final_ms")) + setLongParam(ps, 15, doc.get("bytes")) + + setStrParam(ps, 16, doc.get("referrer")) + setStrParam(ps, 17, doc.get("agent")) + setStrParam(ps, 18, doc.get("session")) + setStrParam(ps, 19, doc.get("visitor_id")) + } + + private static void setStrParam(PreparedStatement ps, int idx, Object val) { + if (val == null) ps.setNull(idx, Types.VARCHAR) else ps.setString(idx, val.toString()) + } + private static void setLongParam(PreparedStatement ps, int idx, Object val) { + if (val instanceof Number) ps.setLong(idx, ((Number) val).longValue()) + else if (val != null) try { ps.setLong(idx, Long.parseLong(val.toString())) } catch (Exception e) { ps.setNull(idx, Types.BIGINT) } + else ps.setNull(idx, Types.BIGINT) } private static void setParam(PreparedStatement ps, int idx, Object value) { diff --git a/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy index 060b66384..5dc28d06e 100644 --- a/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy +++ b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy @@ -195,8 +195,9 @@ class PostgresSearchTranslatorTests { @DisplayName("exists translates to JSONB document ? field") void exists_translatesJsonbHasKey() { QueryResult qr = ElasticQueryTranslator.translateQuery([exists: [field: "email"]]) - Assertions.assertTrue(qr.clause.contains("document ?"), "should use JSONB ? operator") - Assertions.assertTrue(qr.clause.contains("email")) + Assertions.assertTrue(qr.clause.contains("document ??"), "should use JSONB ? operator (escaped as ?? for JDBC)") + Assertions.assertEquals(1, qr.params.size(), "should have 1 param for field name") + Assertions.assertEquals("email", qr.params[0]) } // ============================================================ From f3e01ebdb36202092c418e66c7d0689ad8cc208c Mon Sep 17 00:00:00 2001 From: pandor4u <103976470+pandor4u@users.noreply.github.com> Date: Fri, 10 Apr 2026 18:25:35 -0500 Subject: [PATCH 3/3] feat: Optional TimescaleDB hypertables for log tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the TimescaleDB extension is available, automatically: - Convert moqui_logs and moqui_http_log to hypertables (time-partitioned by log_timestamp) for efficient chunk-level operations - Enable compression on chunks older than 7 days (90%+ storage reduction) with segmentby on log_level (logs) and request_path (http_log) - Add automatic retention policies to drop chunks older than 90 days, replacing Moqui's nightly row-by-row DELETE cleanup jobs - Use drop_chunks() in deleteByQuery for O(1) chunk drops instead of O(n) per-row deletes when TimescaleDB is active Falls back gracefully to plain tables with BRIN indexes when TimescaleDB is not installed. Reports TimescaleDB status in getServerInfo() response. Safe to run multiple times — skips already-converted hypertables. --- .../impl/context/PostgresElasticClient.groovy | 119 +++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy index 11d09bdff..3f3b16c40 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy @@ -60,6 +60,9 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { /** Jackson mapper shared with ElasticFacadeImpl */ static final ObjectMapper jacksonMapper = ElasticFacadeImpl.jacksonMapper + /** Whether the TimescaleDB extension is available for log table hypertables */ + private boolean hasTimescaleDb = false + /** SQL for inserting HTTP request logs into the dedicated moqui_http_log table */ static final String HTTP_LOG_INSERT_SQL = """ INSERT INTO moqui_http_log (log_timestamp, remote_ip, remote_user, server_ip, content_type, @@ -121,6 +124,16 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { try { stmt.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") } catch (Exception extEx) { logger.warn("Could not create pg_trgm extension (may require superuser): ${extEx.message}") } + // Detect TimescaleDB extension for log table hypertables (optional) + try { + stmt.execute("CREATE EXTENSION IF NOT EXISTS timescaledb") + hasTimescaleDb = true + logger.info("TimescaleDB extension detected — log table hypertables enabled") + } catch (Exception extEx) { + hasTimescaleDb = false + logger.info("TimescaleDB extension not available — using plain tables for logs") + } + // moqui_search_index — index metadata (replaces ES index/alias concept) stmt.execute(""" CREATE TABLE IF NOT EXISTS moqui_search_index ( @@ -243,6 +256,11 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { END \$\$; """.trim()) + // Convert log tables to TimescaleDB hypertables with compression and retention + if (hasTimescaleDb) { + initTimescaleHypertables(stmt) + } + logger.info("PostgresElasticClient schema initialized for cluster '${clusterName}'") } finally { stmt.close() @@ -254,6 +272,67 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { } } + /** + * Convert log tables to TimescaleDB hypertables and add compression/retention policies. + * Hypertables partition data into time-based chunks for efficient: + * - Compression (90%+ storage reduction for old chunks) + * - Retention (drop_chunks replaces row-by-row DELETE for cleanup) + * - Query performance (chunk exclusion for time-range queries) + * Safe to call multiple times — skips if tables are already hypertables. + */ + private void initTimescaleHypertables(Statement stmt) { + // Convert moqui_logs to hypertable + try { + // Check if already a hypertable + ResultSet rs = stmt.executeQuery( + "SELECT 1 FROM timescaledb_information.hypertables WHERE hypertable_name = 'moqui_logs'") + boolean isHyper = rs.next() + rs.close() + if (!isHyper) { + stmt.execute("SELECT create_hypertable('moqui_logs', 'log_timestamp', migrate_data => true, if_not_exists => true)") + logger.info("Converted moqui_logs to TimescaleDB hypertable") + } + // Enable compression on chunks older than 7 days + try { + stmt.execute("ALTER TABLE moqui_logs SET (timescaledb.compress, timescaledb.compress_segmentby = 'log_level')") + stmt.execute("SELECT add_compression_policy('moqui_logs', INTERVAL '7 days', if_not_exists => true)") + logger.info("Added compression policy for moqui_logs (compress after 7 days)") + } catch (Exception ex) { logger.debug("Could not set compression on moqui_logs: ${ex.message}") } + // Retention policy: automatically drop chunks older than 90 days + try { + stmt.execute("SELECT add_retention_policy('moqui_logs', INTERVAL '90 days', if_not_exists => true)") + logger.info("Added retention policy for moqui_logs (drop after 90 days)") + } catch (Exception ex) { logger.debug("Could not set retention on moqui_logs: ${ex.message}") } + } catch (Exception ex) { + logger.warn("Could not convert moqui_logs to hypertable: ${ex.message}") + } + + // Convert moqui_http_log to hypertable + try { + ResultSet rs = stmt.executeQuery( + "SELECT 1 FROM timescaledb_information.hypertables WHERE hypertable_name = 'moqui_http_log'") + boolean isHyper = rs.next() + rs.close() + if (!isHyper) { + stmt.execute("SELECT create_hypertable('moqui_http_log', 'log_timestamp', migrate_data => true, if_not_exists => true)") + logger.info("Converted moqui_http_log to TimescaleDB hypertable") + } + // Enable compression on chunks older than 7 days + try { + stmt.execute("ALTER TABLE moqui_http_log SET (timescaledb.compress, timescaledb.compress_segmentby = 'request_path')") + stmt.execute("SELECT add_compression_policy('moqui_http_log', INTERVAL '7 days', if_not_exists => true)") + logger.info("Added compression policy for moqui_http_log (compress after 7 days)") + } catch (Exception ex) { logger.debug("Could not set compression on moqui_http_log: ${ex.message}") } + // Retention policy: automatically drop chunks older than 90 days + try { + stmt.execute("SELECT add_retention_policy('moqui_http_log', INTERVAL '90 days', if_not_exists => true)") + logger.info("Added retention policy for moqui_http_log (drop after 90 days)") + } catch (Exception ex) { logger.debug("Could not set retention on moqui_http_log: ${ex.message}") } + } catch (Exception ex) { + logger.warn("Could not convert moqui_http_log to hypertable: ${ex.message}") + } + } + /** * Get a JDBC Connection from the entity facade for the configured datasource group. * The returned Connection is a Moqui ConnectionWrapper that is transaction-managed. @@ -279,11 +358,13 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { if (rs.next()) { return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres", number: rs.getString(1)], - tagline: "Moqui PostgresElasticClient"] + tagline: "Moqui PostgresElasticClient", + features: [timescaledb: hasTimescaleDb]] } } finally { rs.close() } } finally { ps.close() } - return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres"]] + return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres"], + features: [timescaledb: hasTimescaleDb]] } // ============================================================ @@ -523,6 +604,10 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_http_log") try { return ps.executeUpdate() } finally { ps.close() } } + // When TimescaleDB is available and we have an upper bound, use drop_chunks for efficiency + if (hasTimescaleDb && (range.lte != null || range.lt != null)) { + return dropChunksForTable("moqui_http_log", range) + } List conditions = [] List params = [] if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } @@ -546,6 +631,10 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_logs") try { return ps.executeUpdate() } finally { ps.close() } } + // When TimescaleDB is available and we have an upper bound, use drop_chunks for efficiency + if (hasTimescaleDb && (range.lte != null || range.lt != null)) { + return dropChunksForTable("moqui_logs", range) + } List conditions = [] List params = [] if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } @@ -561,6 +650,32 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { } finally { ps.close() } } + /** + * Use TimescaleDB drop_chunks to efficiently delete old data. + * drop_chunks drops entire compressed chunks at once — O(1) per chunk vs O(n) per row. + * Returns -1 since drop_chunks doesn't report row count. + */ + private Integer dropChunksForTable(String tableName, TimestampRange range) { + Timestamp olderThan = range.lte ?: range.lt + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT drop_chunks('${tableName}', older_than => ?)") + try { + ps.setTimestamp(1, olderThan) + ps.executeQuery().close() + logger.info("drop_chunks('${tableName}', older_than => ${olderThan}) completed") + return -1 // drop_chunks doesn't return a row count + } catch (Exception e) { + logger.warn("drop_chunks failed for ${tableName}, falling back to DELETE: ${e.message}") + // Fall back to regular DELETE + String where = range.lte != null ? "log_timestamp <= ?" : "log_timestamp < ?" + PreparedStatement delPs = conn.prepareStatement("DELETE FROM ${tableName} WHERE ${where}") + try { + delPs.setTimestamp(1, olderThan) + return delPs.executeUpdate() + } finally { delPs.close() } + } finally { ps.close() } + } + /** Simple holder for timestamp range bounds extracted from query DSL */ private static class TimestampRange { Timestamp lte, lt, gte, gt