Skip to content

Commit 2553448

Browse files
pandor4upandor4u
committed
feat: PostgreSQL-backed search/document storage as ElasticSearch alternative
Simplified PR addressing review feedback: - Consolidated duplicated INSERT...ON CONFLICT SQL into shared constant - Removed fork-specific files (AGENTS.md, CLAUDE.md, GEMINI.md, docker compose, .gitignore) - Moved PostgreSQL tests to separate opt-in suite (MoquiSuite untouched) - Updated ElasticRequestLogFilter to use ElasticClient interface type - 83 tests: 46 unit (query translation + SQL injection) + 37 integration New files: - PostgresElasticClient.groovy: Full ElasticClient impl using JSONB/tsvector - ElasticQueryTranslator.groovy: ES Query DSL to PostgreSQL SQL - PostgresSearchLogger.groovy: Log4j2 appender for PostgreSQL - SearchEntities.xml: Entity definitions for search tables - PostgresSearchSuite.groovy: Separate JUnit test suite - PostgresSearchTranslatorTests.groovy: Unit tests - PostgresElasticClientTests.groovy: Integration tests Modified files: - ElasticFacadeImpl.groovy: type=postgres routing - ElasticRequestLogFilter.groovy: Interface type usage - MoquiDefaultConf.xml: Postgres config + entity load - moqui-conf-3.xsd: type attribute with elastic/postgres enum - build.gradle: Test dependencies and suite include
1 parent d12a86e commit 2553448

12 files changed

Lines changed: 3491 additions & 31 deletions

‎framework/build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,8 @@ dependencies {
171171
testImplementation 'org.junit.platform:junit-platform-suite:6.0.1'
172172
// junit-jupiter-api for using JUnit directly, not generally needed for Spock based tests
173173
testImplementation 'org.junit.jupiter:junit-jupiter-api:6.0.1'
174+
// junit-jupiter-engine required to execute @Test-annotated methods via JUnit Platform
175+
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:6.0.1'
174176
// Spock Framework
175177
testImplementation platform('org.spockframework:spock-bom:2.4-groovy-5.0') // Apache 2.0
176178
testImplementation 'org.spockframework:spock-core:2.4-groovy-5.0' // Apache 2.0
@@ -201,6 +203,7 @@ test {
201203

202204
dependsOn cleanTest
203205
include '**/*MoquiSuite.class'
206+
include '**/*PostgresSearchSuite.class'
204207

205208
systemProperty 'moqui.runtime', '../runtime'
206209
systemProperty 'moqui.conf', 'conf/MoquiDevConf.xml'

‎framework/entity/SearchEntities.xml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
This software is in the public domain under CC0 1.0 Universal plus a
4+
Grant of Patent License.
5+
6+
To the extent possible under law, the author(s) have dedicated all
7+
copyright and related and neighboring rights to this software to the
8+
public domain worldwide. This software is distributed without any
9+
warranty.
10+
11+
You should have received a copy of the CC0 Public Domain Dedication
12+
along with this software (see the LICENSE.md file). If not, see
13+
<http://creativecommons.org/publicdomain/zero/1.0/>.
14+
-->
15+
<!--
16+
PostgreSQL-backed search and logging entities.
17+
These tables are created by PostgresElasticClient.initSchema() at startup when type=postgres is configured.
18+
Entity definitions here are provided for Moqui entity framework access (queries, etc) — they reference the
19+
same underlying tables. The actual CREATE TABLE SQL includes JSONB columns and GIN/BRIN indexes that go
20+
beyond what Moqui entities can express, so the DDL is managed by PostgresElasticClient directly.
21+
-->
22+
<entities xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23+
xsi:noNamespaceSchemaLocation="http://moqui.org/xsd/entity-definition-3.xsd">
24+
25+
<!-- ======================================================
26+
Postgres Search — Document Store (replaces ES indexes)
27+
====================================================== -->
28+
29+
<!-- moqui_search_index tracks index metadata (equivalent to ES index aliases + mappings storage) -->
30+
<!-- authorize-skip="create" allows the internal PostgresElasticClient to create records without
31+
per-user authorization. These entities should NOT be exposed via entity-level REST APIs
32+
without additional access controls. -->
33+
<entity entity-name="SearchIndex" package="moqui.search" table-name="moqui_search_index"
34+
group="transactional" use="configuration" cache="never" authorize-skip="create">
35+
<field name="indexName" type="text-medium" is-pk="true"/>
36+
<field name="aliasName" type="text-medium"/>
37+
<field name="docType" type="text-medium"/>
38+
<!-- mapping and settings stored as JSON text; actual column is JSONB in PostgreSQL -->
39+
<field name="mappingJson" type="text-very-long" column-name="mapping"/>
40+
<field name="settingsJson" type="text-very-long" column-name="settings"/>
41+
<field name="createdStamp" type="date-time" default="ec.user.nowTimestamp"/>
42+
</entity>
43+
44+
<!-- moqui_document is the main document store for DataDocuments indexed for search -->
45+
<entity entity-name="SearchDocument" package="moqui.search" table-name="moqui_document"
46+
group="transactional" use="transactional" cache="never" authorize-skip="create">
47+
<field name="indexName" type="text-medium" is-pk="true"/>
48+
<field name="documentId" type="text-medium" is-pk="true" column-name="doc_id"/>
49+
<field name="docType" type="text-medium"/>
50+
<!-- document stored as JSON text; actual column is JSONB in PostgreSQL with GIN index -->
51+
<field name="documentJson" type="text-very-long" column-name="document"/>
52+
<!-- content_text is the extracted text for full-text search; content_tsv is a GENERATED ALWAYS AS tsvector column — not mapped here since Moqui can't express it, but it exists in the DB -->
53+
<field name="contentText" type="text-very-long" column-name="content_text"/>
54+
<field name="createdStamp" type="date-time" default="ec.user.nowTimestamp" column-name="created_stamp"/>
55+
<field name="updatedStamp" type="date-time" column-name="updated_stamp"/>
56+
<index name="SRCH_DOC_TYPE" unique="false"><index-field name="docType"/></index>
57+
</entity>
58+
59+
<!-- ======================================================
60+
Postgres Search — Application Log (replaces moqui_logs ES index)
61+
====================================================== -->
62+
63+
<entity entity-name="SearchLog" package="moqui.search" table-name="moqui_logs"
64+
group="transactional" use="transactional" cache="never"
65+
sequence-bank-size="100" authorize-skip="create">
66+
<field name="logId" type="number-integer" is-pk="true" column-name="log_id"/>
67+
<field name="logTimestamp" type="date-time" column-name="log_timestamp"/>
68+
<field name="logLevel" type="text-short" column-name="log_level"/>
69+
<field name="threadName" type="text-short" column-name="thread_name"/>
70+
<field name="threadId" type="number-integer" column-name="thread_id"/>
71+
<field name="threadPriority" type="number-integer" column-name="thread_priority"/>
72+
<field name="loggerName" type="text-medium" column-name="logger_name"/>
73+
<field name="message" type="text-very-long"/>
74+
<field name="sourceHost" type="text-short" column-name="source_host"/>
75+
<field name="userId" type="id" column-name="user_id"/>
76+
<field name="visitorId" type="id" column-name="visitor_id"/>
77+
<!-- mdc and thrown stored as JSON text; actual columns are JSONB in PostgreSQL -->
78+
<field name="mdcJson" type="text-long" column-name="mdc"/>
79+
<field name="thrownJson" type="text-very-long" column-name="thrown"/>
80+
<index name="MQLOGS_TS" unique="false"><index-field name="logTimestamp"/></index>
81+
<index name="MQLOGS_LVL" unique="false"><index-field name="logLevel"/></index>
82+
</entity>
83+
84+
<!-- ======================================================
85+
Postgres Search — HTTP Request Log (replaces moqui_http_log ES index)
86+
====================================================== -->
87+
88+
<entity entity-name="SearchHttpLog" package="moqui.search" table-name="moqui_http_log"
89+
group="transactional" use="transactional" cache="never"
90+
sequence-bank-size="100" authorize-skip="create">
91+
<field name="logId" type="number-integer" is-pk="true" column-name="log_id"/>
92+
<field name="logTimestamp" type="date-time" column-name="log_timestamp"/>
93+
<field name="remoteIp" type="text-short" column-name="remote_ip"/>
94+
<field name="remoteUser" type="text-short" column-name="remote_user"/>
95+
<field name="serverIp" type="text-short" column-name="server_ip"/>
96+
<field name="contentType" type="text-medium" column-name="content_type"/>
97+
<field name="requestMethod" type="text-short" column-name="request_method"/>
98+
<field name="requestScheme" type="text-short" column-name="request_scheme"/>
99+
<field name="requestHost" type="text-short" column-name="request_host"/>
100+
<field name="requestPath" type="text-medium" column-name="request_path"/>
101+
<field name="requestQuery" type="text-long" column-name="request_query"/>
102+
<field name="httpVersion" type="text-short" column-name="http_version"/>
103+
<field name="responseCode" type="number-integer" column-name="response_code"/>
104+
<field name="timeInitialMs" type="number-integer" column-name="time_initial_ms"/>
105+
<field name="timeFinalMs" type="number-integer" column-name="time_final_ms"/>
106+
<field name="bytesSent" type="number-integer" column-name="bytes_sent"/>
107+
<field name="referrer" type="text-medium"/>
108+
<field name="agent" type="text-medium"/>
109+
<field name="sessionId" type="id" column-name="session_id"/>
110+
<field name="visitorId" type="id" column-name="visitor_id"/>
111+
<index name="MQHTTPLOG_TS" unique="false"><index-field name="logTimestamp"/></index>
112+
<index name="MQHTTPLOG_PATH" unique="false"><index-field name="requestPath"/></index>
113+
</entity>
114+
115+
</entities>

‎framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import org.moqui.impl.entity.EntityDefinition
3232
import org.moqui.impl.entity.EntityJavaUtil
3333
import org.moqui.impl.entity.FieldInfo
3434
import org.moqui.impl.util.ElasticSearchLogger
35+
import org.moqui.impl.util.PostgresSearchLogger
3536
import org.moqui.util.LiteStringMap
3637
import org.moqui.util.MNode
3738
import org.moqui.util.RestClient
@@ -69,8 +70,9 @@ class ElasticFacadeImpl implements ElasticFacade {
6970
}
7071

7172
public final ExecutionContextFactoryImpl ecfi
72-
private final Map<String, ElasticClientImpl> clientByClusterName = new LinkedHashMap<>()
73+
private final Map<String, ElasticClient> clientByClusterName = new LinkedHashMap<>()
7374
private ElasticSearchLogger esLogger = null
75+
private PostgresSearchLogger pgLogger = null
7476

7577
ElasticFacadeImpl(ExecutionContextFactoryImpl ecfi) {
7678
this.ecfi = ecfi
@@ -90,14 +92,21 @@ class ElasticFacadeImpl implements ElasticFacade {
9092
logger.warn("ElasticFacade Client for cluster ${clusterName} already initialized, skipping")
9193
continue
9294
}
93-
if (!clusterUrl) {
94-
logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping")
95-
continue
96-
}
9795

96+
String clusterType = clusterNode.attribute("type") ?: "elastic"
9897
try {
99-
ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi)
100-
clientByClusterName.put(clusterName, elci)
98+
if ("postgres".equals(clusterType)) {
99+
PostgresElasticClient pgc = new PostgresElasticClient(clusterNode, ecfi)
100+
clientByClusterName.put(clusterName, pgc)
101+
logger.info("Initialized PostgresElasticClient for cluster ${clusterName}")
102+
} else {
103+
if (!clusterUrl) {
104+
logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping")
105+
continue
106+
}
107+
ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi)
108+
clientByClusterName.put(clusterName, elci)
109+
}
101110
} catch (Throwable t) {
102111
Throwable cause = t.getCause()
103112
if (cause != null && cause.message.contains("refused")) {
@@ -108,22 +117,29 @@ class ElasticFacadeImpl implements ElasticFacade {
108117
}
109118
}
110119

111-
// init ElasticSearchLogger
112-
if (esLogger == null || !esLogger.isInitialized()) {
113-
ElasticClientImpl loggerEci = clientByClusterName.get("logger") ?: clientByClusterName.get("default")
114-
if (loggerEci != null) {
115-
logger.info("Initializing ElasticSearchLogger with cluster ${loggerEci.getClusterName()}")
116-
esLogger = new ElasticSearchLogger(loggerEci, ecfi)
120+
// init ElasticSearchLogger / PostgresSearchLogger depending on backend type
121+
ElasticClient loggerClient = clientByClusterName.get("logger") ?: clientByClusterName.get("default")
122+
if (loggerClient instanceof PostgresElasticClient) {
123+
if (pgLogger == null || !pgLogger.isInitialized()) {
124+
logger.info("Initializing PostgresSearchLogger with cluster ${loggerClient.getClusterName()}")
125+
pgLogger = new PostgresSearchLogger((PostgresElasticClient) loggerClient, ecfi)
126+
} else {
127+
logger.warn("PostgresSearchLogger in place and initialized, skipping")
128+
}
129+
} else if (loggerClient instanceof ElasticClientImpl) {
130+
if (esLogger == null || !esLogger.isInitialized()) {
131+
logger.info("Initializing ElasticSearchLogger with cluster ${loggerClient.getClusterName()}")
132+
esLogger = new ElasticSearchLogger((ElasticClientImpl) loggerClient, ecfi)
117133
} else {
118-
logger.warn("No Elastic Client found with name 'logger' or 'default', not initializing ElasticSearchLogger")
134+
logger.warn("ElasticSearchLogger in place and initialized, skipping")
119135
}
120136
} else {
121-
logger.warn("ElasticSearchLogger in place and initialized, not initializing ElasticSearchLogger")
137+
logger.warn("No Elastic/Postgres Client found with name 'logger' or 'default', not initializing search logger")
122138
}
123139

124140
// Index DataFeed with indexOnStartEmpty=Y
125141
try {
126-
ElasticClientImpl defaultEci = clientByClusterName.get("default")
142+
ElasticClient defaultEci = clientByClusterName.get("default")
127143
if (defaultEci != null) {
128144
EntityList dataFeedList = ecfi.entityFacade.find("moqui.entity.feed.DataFeed")
129145
.condition("indexOnStartEmpty", "Y").disableAuthz().list()
@@ -151,7 +167,11 @@ class ElasticFacadeImpl implements ElasticFacade {
151167

152168
void destroy() {
153169
if (esLogger != null) esLogger.destroy()
154-
for (ElasticClientImpl eci in clientByClusterName.values()) eci.destroy()
170+
if (pgLogger != null) pgLogger.destroy()
171+
for (ElasticClient eci in clientByClusterName.values()) {
172+
if (eci instanceof ElasticClientImpl) ((ElasticClientImpl) eci).destroy()
173+
else if (eci instanceof PostgresElasticClient) ((PostgresElasticClient) eci).destroy()
174+
}
155175
}
156176

157177
@Override ElasticClient getDefault() { return clientByClusterName.get("default") }

0 commit comments

Comments
 (0)