#106

adds support for highlighting, not yet forwarded to FR

Author: robfrank

src/main/java/com/arcadeanalytics/index/ElasticBulkGraphIndexer.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -101,6 +101,7 @@ public ElasticBulkGraphIndexer(Client client, String indexName) {
                         "             \"mapping\": {\n" +
                         "                \"type\": \"string\",\n" +
                         "                \"analyzer\": \"standard\",\n" +
+                        "                \"store\": \"yes\",\n" +
                         "                \"term_vector\": \"with_positions_offsets\",\n" +
                         "                \"fields\": {\n" +
                         "                   \"raw\": {\n" +
@@ -125,6 +126,7 @@ public ElasticBulkGraphIndexer(Client client, String indexName) {
                         "             \"mapping\": {\n" +
                         "                \"type\": \"string\",\n" +
                         "                \"analyzer\": \"standard\",\n" +
+                        "                \"store\": \"yes\",\n" +
                         "                \"term_vector\": \"with_positions_offsets\",\n" +
                         "                \"fields\": {\n" +
                         "                   \"raw\": {\n" +

src/main/java/com/arcadeanalytics/repository/DataSourceRepository.java

@@ -28,6 +28,7 @@
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
+import java.util.Optional;
 
 
 /**
@@ -37,6 +38,8 @@
 @Repository
 public interface DataSourceRepository extends JpaRepository<DataSource, Long> {
 
+    Optional<DataSource> findByName(String name);
+
     Page<DataSource> findByWorkspaceUserUserLogin(String currentUserLogin, Pageable pageable);
 
     List<DataSource> findByWorkspaceUserUserLogin(String currentUserLogin);

src/main/java/com/arcadeanalytics/service/ElasticGraphIndexerService.java

@@ -46,6 +46,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -219,26 +220,28 @@ public List<Sprite> search(DataSource dataSource, SearchQueryDTO query) throws I
         final String indexName = dataSource.getId().toString();
 
         SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName)
-//                .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
-                .setQuery(QueryBuilders.queryStringQuery(query.getQuery())
-                        .defaultOperator(AND))
                 .setHighlighterForceSource(true)
                 .setHighlighterEncoder("default")
                 .setHighlighterPreTags("<em>")
                 .setHighlighterPostTags("</em>")
                 .setSize(query.getNumOfDocuments());
 
+        final QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query.getQuery()).defaultOperator(AND);
+
         indexFields(indexName, client, query.isUseEdges())
                 .stream()
                 .forEach(field -> {
-                    searchRequestBuilder.addHighlightedField(field,-1,-1);
-                    searchRequestBuilder.addHighlightedField(field + ".raw",-1,-1);
+                    searchRequestBuilder.addHighlightedField(field);
+                    queryBuilder.field(field);
+//                    searchRequestBuilder.addHighlightedField(field + ".raw");
                 });
 
         if (query.getIds().length > 0) {
             searchRequestBuilder.setPostFilter(QueryBuilders.termsQuery(ARCADE_ID, query.getIds()));
         }
 
+        searchRequestBuilder.setQuery(queryBuilder);
+
         log.debug("query:: {}", searchRequestBuilder.toString());
         SearchResponse searchResponse = searchRequestBuilder
                 .execute()
@@ -253,9 +256,14 @@ public List<Sprite> search(DataSource dataSource, SearchQueryDTO query) throws I
         return Stream.of(hits)
 //                .peek(h -> log.info("h:: " + h.toString()))
 //                .peek(h -> log.info("hf:: " + h.getHighlightFields().size()))
-                .map(h -> h.getSource())
+//                .map(h -> h.getSource())
 //                .peek(s -> log.info("s:: " + s))
-                .map(s -> new Sprite().load(s))
+                .map(hit -> {
+
+                    Sprite sprite = new Sprite().load(hit.getSource());
+//                    hit.getHighlightFields().f
+                    return sprite;
+                })
                 .collect(Collectors.toList());
 
     }
@@ -283,6 +291,9 @@ public Map<String, Object> aggregate(DataSource dataSource,
 
         final SearchResponse searchResponse = termAggregations(client, indexName, classes, fields, query, minDocCount, maxValuesPerField);
 
+
+        log.debug("tree: {}", searchResponse.toString());
+
         Map<String, Object> facetsTree = searchResponseToMap(fields, searchResponse);
 
         log.info("done aggregation on data-source {}", dataSource.getId(), minDocCount, maxValuesPerField);
@@ -310,7 +321,7 @@ public Map<String, Object> aggregate(DataSource dataSource,
 
         Map<String, Object> facetsTree = searchResponseToMap(fields, searchResponse);
 
-        log.info("done aggregation on data-source {}", dataSource.getName(), minDocCount, maxValuesPerField);
+        log.info("done aggregation on data-source {}", dataSource.getId(), minDocCount, maxValuesPerField);
 
         return facetsTree;
 
@@ -389,7 +400,7 @@ private SearchResponse termAggregations(Client client,
 
 
         if (query.getIds().length > 0) {
-            log.info("aggregate only on :: {}", truncate(join(",", query.getIds()), 100));
+            log.debug("aggregate only on :: {}", truncate(join(",", query.getIds()), 100));
             searchRequestBuilder.setQuery(QueryBuilders.boolQuery()
                     .filter(QueryBuilders.termsQuery(ARCADE_ID, query.getIds())));
         }

src/test/java/com/arcadeanalytics/service/ElasticGraphIndexerServiceIntTest.java

@@ -27,7 +27,6 @@
 import com.arcadeanalytics.domain.enumeration.DataSourceType;
 import com.arcadeanalytics.repository.DataSourceRepository;
 import com.arcadeanalytics.service.dto.SearchQueryDTO;
-import com.google.common.collect.Sets;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -45,6 +44,7 @@
 import java.util.stream.Collectors;
 
 import static com.arcadeanalytics.index.IndexConstants.ARCADE_ID;
+import static com.google.common.collect.Sets.newHashSet;
 import static java.util.Collections.emptySet;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -76,18 +76,19 @@ public static void beforeClass() {
     @Before
     public void setUp() throws Exception {
         //given
-        dataSource = new DataSource()
-                .name("demodb")
-                .description("desc")
-                .remote(false)
-                .type(DataSourceType.ORIENTDB3)
-                .server(container.getContainerIpAddress())
-                .port(container.getFirstMappedPort())
-                .database("demodb")
-                .username("admin")
-                .password("admin");
-
-        dataSourceRepository.save(dataSource);
+
+        dataSource = dataSourceRepository.findByName("demodb").orElseGet(() ->
+                dataSourceRepository.save(new DataSource()
+                        .name("demodb")
+                        .description("desc")
+                        .remote(false)
+                        .type(DataSourceType.ORIENTDB3)
+                        .server(container.getContainerIpAddress())
+                        .port(container.getFirstMappedPort())
+                        .database("demodb")
+                        .username("admin")
+                        .password("admin"))
+        );
 
         if (!service.hasIndex(dataSource)) {
             //when --> ASYNC!!!!!
@@ -102,6 +103,8 @@ public void setUp() throws Exception {
     @Test(expected = RuntimeException.class)
     public void shouldDeleteIndex() throws IOException {
 
+        assertThat(service.hasIndex(dataSource)).isTrue();
+
         final boolean deleted = service.deleteIndex(dataSource);
 
         assertThat(deleted).isTrue();
@@ -110,6 +113,7 @@ public void shouldDeleteIndex() throws IOException {
         SearchQueryDTO queryDTO = new SearchQueryDTO();
         service.search(dataSource, queryDTO);
 
+        assertThat(service.hasIndex(dataSource)).isFalse();
 
     }
 
@@ -118,9 +122,11 @@ public void shouldFindSameDocumentsWithSimpleSearchAndFilteringOnIds() throws IO
         //then simple search for frank and rob
         SearchQueryDTO queryDTO = new SearchQueryDTO();
         //explicit OR 'cause default search are in AND
-        queryDTO.setQuery("roma OR frank");
+        queryDTO.setQuery("Name:roma OR Name:frank");
         List<Sprite> docs = service.search(dataSource, queryDTO);
-        assertThat(docs).hasSize(10);
+        assertThat(docs).hasSize(9);
+
+//        docs.forEach(sprite -> assertThat(sprite.valueOf("Name")).contains("frank", "roma", "Frank", "Roma"));
 
         String[] ids = docs.stream()
                 .map(s -> s.valueOf(ARCADE_ID))
@@ -132,7 +138,9 @@ public void shouldFindSameDocumentsWithSimpleSearchAndFilteringOnIds() throws IO
         queryDTO.setIds(ids);
         //now search for all documents and filter by previous ids
         docs = service.search(dataSource, queryDTO);
-        assertThat(docs).hasSize(10);
+        assertThat(docs).hasSize(9);
+
+//        docs.forEach(sprite -> assertThat(sprite.valueOf("Name")).contains("frank", "roma", "Frank", "Roma"));
 
     }
 
@@ -146,9 +154,10 @@ public void shouldProvideFacetingOverNodesPropertyValues() throws IOException {
 
         List<Sprite> docs = service.search(dataSource, queryDTO);
 
-        //search limited to 10
+        //search limited to 50
         assertThat(docs).hasSize(50);
 
+        //maps the ids of retrieved documents
         String[] ids = docs.stream()
                 .map(s -> s.valueOf(ARCADE_ID))
                 .collect(Collectors.toList())
@@ -161,7 +170,6 @@ public void shouldProvideFacetingOverNodesPropertyValues() throws IOException {
 
         final Map<String, Object> aggregate = service.aggregate(dataSource, queryDTO, emptySet(), emptySet(), 1, 20);
 
-        System.out.println("aggregate = " + aggregate);
         assertThat(aggregate).containsKeys("Countries");
 
         final Map<String, Object> person = (Map<String, Object>) aggregate.get("Countries");
@@ -199,7 +207,7 @@ public void shouldProvideFacetingOverNodesPropertyValuesWithClassesAndProperty()
         queryDTO.setUseEdges(true);
         queryDTO.setIds(ids);
 
-        final Map<String, Object> aggregate = service.aggregate(dataSource, queryDTO, Sets.newHashSet("Countries"), Sets.newHashSet("Code"), 1, 20);
+        final Map<String, Object> aggregate = service.aggregate(dataSource, queryDTO, newHashSet("Countries"), newHashSet("Code"), 1, 20);
 
         assertThat(aggregate).containsKeys("Countries");