Skip to content

Commit 723146c

Browse files
robfrankrobfrank
committed
adds hasIndex method adds watchdog on multiple indexing requests reviews logs
1 parent 50a87ac commit 723146c

3 files changed

Lines changed: 40 additions & 30 deletions

File tree

‎src/main/java/com/arcadeanalytics/service/ElasticGraphIndexerService.java‎

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public ElasticGraphIndexerService(ElasticsearchTemplate searchTemplate,
121121
@Timed
122122
public Future<DataSourceIndex> index(DataSource datasource, String... queries) {
123123

124-
log.info("start indexing of datasource {} with queries {} ", datasource, queries);
124+
log.info("start indexing of datasource {} with queries {} ", datasource.getId(), queries);
125125

126126
datasource.indexing(IndexingStatus.INDEXING);
127127

@@ -143,35 +143,45 @@ public Future<DataSourceIndex> index(DataSource datasource, String... queries) {
143143
.endedAt(LocalDate.now())
144144
.dataSource(datasource);
145145

146-
dataSourceIndexRepository.save(index);
147-
148146
datasource.indexing(IndexingStatus.INDEXED);
149147

150-
dataSourceRepository.save(datasource);
151-
152-
log.info("end of indexing datasource {} ", datasource);
148+
log.info("end of indexing datasource {} ", datasource.getId());
153149

154150
return new AsyncResult<>(index);
155151
} catch (Exception e) {
156-
log.error("error while indexing datasource " + datasource, e);
152+
log.error("error while indexing datasource " + datasource.getId(), e);
157153

158154
index.documents(0L)
159155
.report("Error while indexing: " + e.getMessage())
160156
.endedAt(LocalDate.now())
161157
.dataSource(datasource);
162158

163-
dataSourceIndexRepository.save(index);
164-
165159
datasource.indexing(IndexingStatus.NOT_INDEXED);
166160

167-
dataSourceRepository.save(datasource);
168161
throw e;
162+
} finally {
163+
dataSourceIndexRepository.save(index);
164+
dataSourceRepository.save(datasource);
165+
169166
}
170167
}
171168

172169

170+
public boolean hasIndex(DataSource dataSource) {
171+
log.info("check index for data-source {} ", dataSource.getId());
172+
173+
final Client client = searchTemplate.getClient();
174+
IndicesAdminClient indices = client.admin().indices();
175+
176+
String indexName = dataSource.getId().toString();
177+
IndicesExistsResponse existsResponse = indices
178+
.exists(Requests.indicesExistsRequest(indexName)).actionGet();
179+
180+
return existsResponse.isExists();
181+
}
182+
173183
public boolean deleteIndex(DataSource dataSource) {
174-
log.info("delete index for data-source {} ", dataSource);
184+
log.info("delete index for data-source {} ", dataSource.getId());
175185

176186
final Client client = searchTemplate.getClient();
177187
IndicesAdminClient indices = client.admin().indices();
@@ -209,15 +219,21 @@ public List<Sprite> search(DataSource dataSource, SearchQueryDTO query) throws I
209219
final String indexName = dataSource.getId().toString();
210220

211221
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName)
212-
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
222+
// .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
213223
.setQuery(QueryBuilders.queryStringQuery(query.getQuery())
214224
.defaultOperator(AND))
215225
.setHighlighterForceSource(true)
216-
.setSize(10);
226+
.setHighlighterEncoder("default")
227+
.setHighlighterPreTags("<em>")
228+
.setHighlighterPostTags("</em>")
229+
.setSize(query.getNumOfDocuments());
217230

218231
indexFields(indexName, client, query.isUseEdges())
219232
.stream()
220-
.forEach(f -> searchRequestBuilder.addHighlightedField(f));
233+
.forEach(field -> {
234+
searchRequestBuilder.addHighlightedField(field,-1,-1);
235+
searchRequestBuilder.addHighlightedField(field + ".raw",-1,-1);
236+
});
221237

222238
if (query.getIds().length > 0) {
223239
searchRequestBuilder.setPostFilter(QueryBuilders.termsQuery(ARCADE_ID, query.getIds()));
@@ -228,12 +244,17 @@ public List<Sprite> search(DataSource dataSource, SearchQueryDTO query) throws I
228244
.execute()
229245
.actionGet();
230246

247+
log.debug("response:: {} ", searchResponse.toString());
248+
231249
SearchHit[] hits = searchResponse.getHits().getHits();
232250

233251
log.info("found {} documents", hits.length);
234252

235253
return Stream.of(hits)
254+
// .peek(h -> log.info("h:: " + h.toString()))
255+
// .peek(h -> log.info("hf:: " + h.getHighlightFields().size()))
236256
.map(h -> h.getSource())
257+
// .peek(s -> log.info("s:: " + s))
237258
.map(s -> new Sprite().load(s))
238259
.collect(Collectors.toList());
239260

@@ -250,7 +271,7 @@ public Map<String, Object> aggregate(DataSource dataSource,
250271
final String indexName = dataSource.getId().toString();
251272

252273
log.info("start aggregation on data-source {} with minDocCount {}, maxValuesPerField {}, classes {} , fields {}",
253-
dataSource.getName(),
274+
dataSource.getId(),
254275
minDocCount,
255276
maxValuesPerField,
256277
classes,
@@ -264,7 +285,7 @@ public Map<String, Object> aggregate(DataSource dataSource,
264285

265286
Map<String, Object> facetsTree = searchResponseToMap(fields, searchResponse);
266287

267-
log.info("done aggregation on data-source {}", dataSource.getName(), minDocCount, maxValuesPerField);
288+
log.info("done aggregation on data-source {}", dataSource.getId(), minDocCount, maxValuesPerField);
268289

269290
return facetsTree;
270291

@@ -279,7 +300,7 @@ public Map<String, Object> aggregate(DataSource dataSource,
279300

280301
final String indexName = dataSource.getId().toString();
281302

282-
log.info("start aggregation on data-source {} on classes {} and fields {} with minDocCount {}, maxValuesPerField {}", dataSource.getName(), classes, fields, minDocCount, maxValuesPerField);
303+
log.info("start aggregation on data-source {} on classes {} and fields {} with minDocCount {}, maxValuesPerField {}", dataSource.getId(), classes, fields, minDocCount, maxValuesPerField);
283304

284305
Client client = searchTemplate.getClient();
285306

‎src/main/java/com/arcadeanalytics/service/WidgetService.java‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -588,24 +588,13 @@ private Optional<Widget> getWidgetIfAllowed(Long id, Long userId) {
588588

589589
}
590590

591-
592-
private Contract contractOld(Widget widget) {
593-
594-
final Contract contract = widget.getDashboard().getWorkspace().getUser().getCompany().getContract();
595-
596-
log.info("contrat:: {} ", contract);
597-
return contract;
598-
599-
600-
}
601-
602591
private Contract contract() {
603592

604593
final Contract contract = SecurityUtils.getCurrentUserLogin()
605594
.flatMap(user -> arcadeUserRepository.findByUserLogin(user))
606595
.map(arcadeUser -> arcadeUser.getCompany().getContract()).get();
607596

608-
log.info("contrat:: {} ", contract);
597+
log.debug("contract:: {} ", contract);
609598
return contract;
610599

611600

‎src/main/java/com/arcadeanalytics/web/rest/DataSourceResource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ public ResponseEntity<DataSourceMetadata> getDataSourceMetadata(@PathVariable Lo
228228

229229
DataSourceMetadataProvider provider = dataSourceMetadataProviderFactory.create(dsInfo);
230230
final DataSourceMetadata metadata = provider.fetchMetadata(dsInfo);
231-
log.info("metadata for DataSource : {} - {}", id, metadata);
231+
log.debug("metadata for DataSource : {} - {}", id, metadata);
232232
return ResponseUtil.wrapOrNotFound(Optional.ofNullable(metadata));
233233
}
234234

0 commit comments

Comments
 (0)