Skip to content

Commit 8e725b7

Browse files
skyflow-bhartiskyflow-bharti
authored
Merge pull request #232 from skyflowapi/skyflow-vivek/SK-2292-update-default-concurrency
SK-2292 update default concurrency
2 parents 6f23911 + a25231a commit 8e725b7

5 files changed

Lines changed: 61 additions & 32 deletions

File tree

‎v3/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
</parent>
1212

1313
<artifactId>skyflow-java</artifactId>
14-
<version>2.0.0-beta.4-dev.d3257dc</version>
14+
<version>3.0.0-beta.4</version>
1515
<packaging>jar</packaging>
1616
<name>${project.groupId}:${project.artifactId}</name>
1717
<description>Skyflow V3 SDK for the Java programming language</description>

‎v3/src/main/java/com/skyflow/utils/Constants.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22

33
public final class Constants extends BaseConstants {
44
public static final String SDK_NAME = "Skyflow Java SDK ";
5-
public static final String SDK_VERSION = "3.0.0-beta.3";
5+
public static final String SDK_VERSION = "3.0.0-beta.4";
66
public static final String VAULT_DOMAIN = ".skyvault.";
77
public static final String SDK_PREFIX = SDK_NAME + SDK_VERSION;
88
public static final Integer INSERT_BATCH_SIZE = 50;
99
public static final Integer MAX_INSERT_BATCH_SIZE = 1000;
10-
public static final Integer INSERT_CONCURRENCY_LIMIT = 10;
10+
public static final Integer INSERT_CONCURRENCY_LIMIT = 1;
1111
public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 10;
1212

1313
public static final Integer DETOKENIZE_BATCH_SIZE = 50;
14-
public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 10;
14+
public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 1;
1515
public static final Integer MAX_DETOKENIZE_BATCH_SIZE = 1000;
1616
public static final Integer MAX_DETOKENIZE_CONCURRENCY_LIMIT = 10;
1717

‎v3/src/main/java/com/skyflow/utils/validations/Validations.java‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -93,37 +93,39 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S
9393
if (tokens == null || tokens.isEmpty()) {
9494
LogUtil.printErrorLog(Utils.parameterizedString(
9595
ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
96-
));
97-
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyDetokenizeData.getMessage());
96+
));
97+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyDetokenizeData.getMessage());
9898
}
99-
for (String token : tokens) {
99+
for (int index = 0; index < tokens.size(); index++) {
100+
String token = tokens.get(index);
100101
if (token == null || token.trim().isEmpty()) {
101102
LogUtil.printErrorLog(Utils.parameterizedString(
102-
ErrorLogs.EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
103-
));
103+
ErrorLogs.EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA.getLog(),
104+
InterfaceName.DETOKENIZE.getName(),
105+
String.valueOf(index)));
104106
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyTokenInDetokenizeData.getMessage());
105107
}
106108
}
109+
107110
List<TokenGroupRedactions> groupRedactions = request.getTokenGroupRedactions();
108111
if (groupRedactions != null && !groupRedactions.isEmpty()) {
109-
for (TokenGroupRedactions group : groupRedactions) {
110-
if (group == null) {
111-
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_REDACTION_GROUP_OBJECT.getLog(), InterfaceName.DETOKENIZE.getName()));
112-
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupRedactions.getMessage());
113-
}
114-
String groupName = group.getTokenGroupName();
115-
String redaction = group.getRedaction();
116-
if (groupName == null || groupName.trim().isEmpty()) {
117-
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
118-
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage());
119-
}
120-
if (redaction == null || redaction.trim().isEmpty()) {
121-
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
122-
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullRedactionInTokenGroup.getMessage());
123-
}
112+
for (TokenGroupRedactions group : groupRedactions) {
113+
if (group == null) {
114+
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_REDACTION_GROUP_OBJECT.getLog(), InterfaceName.DETOKENIZE.getName()));
115+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupRedactions.getMessage());
116+
}
117+
String groupName = group.getTokenGroupName();
118+
String redaction = group.getRedaction();
119+
if (groupName == null || groupName.trim().isEmpty()) {
120+
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
121+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage());
122+
}
123+
if (redaction == null || redaction.trim().isEmpty()) {
124+
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
125+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullRedactionInTokenGroup.getMessage());
124126
}
125127
}
126-
128+
}
127129
}
128130

129131
public static void validateVaultConfiguration(VaultConfig vaultConfig) throws SkyflowException {

‎v3/src/main/java/com/skyflow/vault/controller/VaultController.java‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.skyflow.utils.validations.Validations;
2020
import com.skyflow.vault.data.*;
2121
import io.github.cdimascio.dotenv.Dotenv;
22+
import io.github.cdimascio.dotenv.DotenvException;
2223

2324
import java.util.ArrayList;
2425
import java.util.Collections;
@@ -309,9 +310,22 @@ private InsertResponse insertBatch(List<InsertRecordData> batch, String tableNam
309310

310311
private void configureInsertConcurrencyAndBatchSize(int totalRequests) {
311312
try {
312-
Dotenv dotenv = Dotenv.load();
313-
String userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE");
314-
String userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT");
313+
String userProvidedBatchSize = System.getenv("INSERT_BATCH_SIZE");
314+
String userProvidedConcurrencyLimit = System.getenv("INSERT_CONCURRENCY_LIMIT");
315+
316+
Dotenv dotenv = null;
317+
try {
318+
dotenv = Dotenv.load();
319+
} catch (DotenvException ignored) {
320+
// ignore the case if .env file is not found
321+
}
322+
323+
if (userProvidedBatchSize == null && dotenv != null) {
324+
userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE");
325+
}
326+
if (userProvidedConcurrencyLimit == null && dotenv != null) {
327+
userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT");
328+
}
315329

316330
if (userProvidedBatchSize != null) {
317331
try {
@@ -365,9 +379,22 @@ private void configureInsertConcurrencyAndBatchSize(int totalRequests) {
365379

366380
private void configureDetokenizeConcurrencyAndBatchSize(int totalRequests) {
367381
try {
368-
Dotenv dotenv = Dotenv.load();
369-
String userProvidedBatchSize = dotenv.get("DETOKENIZE_BATCH_SIZE");
370-
String userProvidedConcurrencyLimit = dotenv.get("DETOKENIZE_CONCURRENCY_LIMIT");
382+
String userProvidedBatchSize = System.getenv("DETOKENIZE_BATCH_SIZE");
383+
String userProvidedConcurrencyLimit = System.getenv("DETOKENIZE_BATCH_SIZE");
384+
385+
Dotenv dotenv = null;
386+
try {
387+
dotenv = Dotenv.load();
388+
} catch (DotenvException ignored) {
389+
// ignore the case if .env file is not found
390+
}
391+
392+
if (userProvidedBatchSize == null && dotenv != null) {
393+
userProvidedBatchSize = dotenv.get("DETOKENIZE_BATCH_SIZE");
394+
}
395+
if (userProvidedConcurrencyLimit == null && dotenv != null) {
396+
userProvidedConcurrencyLimit = dotenv.get("DETOKENIZE_BATCH_SIZE");
397+
}
371398

372399
if (userProvidedBatchSize != null) {
373400
try {

‎v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,7 @@ public void testFractionalLastBatch() throws Exception {
395395

396396
// Last batch should have 50 records, concurrency should be 101
397397
assertEquals(100, getPrivateInt(controller, "insertBatchSize"));
398-
assertEquals(10, getPrivateInt(controller, "insertConcurrencyLimit"));
398+
assertEquals(Constants.INSERT_CONCURRENCY_LIMIT.intValue(), getPrivateInt(controller, "insertConcurrencyLimit"));
399399
}
400400

401401
private int getPrivateInt(Object obj, String field) throws Exception {

0 commit comments

Comments
 (0)