Skip to content

Optimize the Elasticsearch for better Response Time #131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
vanitha1822 wants to merge 2 commits into
base: release-3.6.1
Choose a base branch
from
Draft

Optimize the Elasticsearch for better Response Time #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dbb0a79
fix: optimize the index and reduce the size
vanitha1822 Jan 13, 2026
50d123c
fix: align indent
vanitha1822 Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 86 additions & 11 deletions ...ain/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.mapping.*;
import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
import co.elastic.clients.elasticsearch.indices.IndexSettings;
import co.elastic.clients.elasticsearch.indices.TranslogDurability;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -27,7 +30,7 @@ public class ElasticsearchIndexingService {
private String beneficiaryIndex;

/**
* Create or recreate the Elasticsearch index with proper mapping

*/
public void createIndexWithMapping() throws Exception {
logger.info("Creating index with mapping: {}", beneficiaryIndex);
Expand All @@ -38,24 +41,71 @@ public void createIndexWithMapping() throws Exception {
esClient.indices().delete(d -> d.index(beneficiaryIndex));
}

// Create index with mapping
IndexSettings settings = IndexSettings.of(s -> s
.refreshInterval(t -> t.time("30s"))

.numberOfShards("3")

.numberOfReplicas("1")

.queries(q -> q
.cache(c -> c.enabled(true))
)

.maxResultWindow(10000)

.translog(t -> t
.durability(TranslogDurability.Async)
.syncInterval(ts -> ts.time("30s"))
)
);

TypeMapping mapping = TypeMapping.of(tm -> tm
.properties("benId", Property.of(p -> p.keyword(k -> k)))
.properties("benRegId", Property.of(p -> p.long_(l -> l)))

.properties("beneficiaryID", Property.of(p -> p.keyword(k -> k)))

.properties("firstName", Property.of(p -> p.text(t -> t
.fields("keyword", Property.of(fp -> fp.keyword(k -> k))))))
.analyzer("standard")
.fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256))))
.fields("prefix", Property.of(fp -> fp.text(txt -> txt
.analyzer("standard")
.indexPrefixes(ip -> ip.minChars(2).maxChars(5)) // Fast prefix search
)))
)))

.properties("lastName", Property.of(p -> p.text(t -> t
.fields("keyword", Property.of(fp -> fp.keyword(k -> k))))))
.analyzer("standard")
.fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256))))
.fields("prefix", Property.of(fp -> fp.text(txt -> txt
.analyzer("standard")
.indexPrefixes(ip -> ip.minChars(2).maxChars(5))
)))
)))

.properties("fatherName", Property.of(p -> p.text(t -> t
.analyzer("standard")
.fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256))))
)))

.properties("spouseName", Property.of(p -> p.text(t -> t
.analyzer("standard")
.fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256))))
)))

.properties("genderID", Property.of(p -> p.integer(i -> i)))
.properties("genderName", Property.of(p -> p.keyword(k -> k)))
.properties("dOB", Property.of(p -> p.date(d -> d)))
.properties("dOB", Property.of(p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis"))))
.properties("age", Property.of(p -> p.integer(i -> i)))
.properties("phoneNum", Property.of(p -> p.keyword(k -> k)))
.properties("fatherName", Property.of(p -> p.text(t -> t
.fields("keyword", Property.of(fp -> fp.keyword(k -> k))))))
.properties("spouseName", Property.of(p -> p.text(t -> t
.fields("keyword", Property.of(fp -> fp.keyword(k -> k))))))

.properties("phoneNum", Property.of(p -> p.keyword(k -> k
.fields("ngram", Property.of(fp -> fp.text(txt -> txt
.analyzer("standard")
.searchAnalyzer("standard")
)))
)))

.properties("isHIVPos", Property.of(p -> p.keyword(k -> k)))
.properties("createdBy", Property.of(p -> p.keyword(k -> k)))
.properties("createdDate", Property.of(p -> p.date(d -> d)))
Expand Down Expand Up @@ -94,22 +144,47 @@ public void createIndexWithMapping() throws Exception {

esClient.indices().create(c -> c
.index(beneficiaryIndex)
.settings(settings)
.mappings(mapping)
);

logger.info("Index created successfully: {}", beneficiaryIndex);
}

/**
* Reset refresh interval after bulk indexing completes
* Call this after syncAllBeneficiaries() finishes
*/
public void optimizeForSearch() throws Exception {
logger.info("Optimizing index for search performance...");

esClient.indices().putSettings(s -> s
.index(beneficiaryIndex)
.settings(is -> is
.refreshInterval(t -> t.time("1s"))
.translog(t -> t.durability(TranslogDurability.Request))
)
);

esClient.indices().forcemerge(f -> f
.index(beneficiaryIndex)
.maxNumSegments(1L) // Optimal for read-heavy workloads
);

}

/**
* Index all beneficiaries - delegates to existing sync service
* This is much safer than loading all records at once
*/
public Map<String, Integer> indexAllBeneficiaries() {
logger.info("Starting full indexing via sync service...");

try {
ElasticsearchSyncService.SyncResult result = syncService.syncAllBeneficiaries();

// After indexing completes, optimize for search
optimizeForSearch();

Map<String, Integer> response = new HashMap<>();
response.put("success", result.getSuccessCount());
response.put("failed", result.getFailureCount());
Expand Down
Loading