feat(spring-batch-file-example): implement Spring Batch file processi…

spring-batch-file-examples/README.md

@@ -1,22 +1,41 @@
-# Spring Batch Examples | DB And Async 
+# Spring Batch File Readers
 
-This project is a **Spring Boot** application demonstrating a **fully asynchronous Spring Batch job**, designed with a focus on **performance** and **scalability**.
+This project is a **Spring Boot** application that demonstrates how to build **custom file readers using Spring Batch**, with a strong focus on **performance**, **scalability**, and **clean design**.
+
+The main goal of this repository is to showcase **different strategies for reading files** depending on their size and characteristics, following **real-world batch processing patterns**.
 
 ---
 
 ## 🚀 Overview
 
-The example showcases how to configure and run an **asynchronous Spring Batch job** that processes a large dataset efficiently.  
-The job reads **10,000 records** from a database table, simulating item processing by printing  
-`"item processed"` for each entry.
+The project currently provides **custom Spring Batch `ItemReader` implementations** for reading Excel files, using **different approaches for small and large files**:
+
+- **Small Excel files**: loaded and processed entirely in memory
+- **Large Excel files**: streamed row by row to minimize memory usage
+
+The architecture is intentionally extensible, allowing additional file formats (such as **CSV**) to be added in the future without changing the core batch flow.
 
 ---
 
-## ⚙️ How It Works
+## 📂 Supported File Types
+
+### ✅ Currently Implemented
+
+- **Small Excel files (`.xlsx`)**
+    - Suitable for files that fit comfortably in memory
+    - Simple and fast processing
+
+- **Large Excel files (`.xlsx`)**
+    - Streaming-based reader
+    - Designed for large datasets
+    - Low memory footprint
+    - Handles empty rows gracefully
+
+### 🕒 Planned
+
+- **CSV files**
+- Other structured file formats (as needed)
 
-- The job leverages Spring Batch’s asynchronous capabilities to read and process data concurrently.
-- An **H2 in-memory database** is used to store the sample data.
-- The asynchronous behavior is enabled through a specific Spring profile.
 
 ---
 
@@ -25,6 +44,13 @@ The job reads **10,000 records** from a database table, simulating item processi
 - **Java 21**
 - **Spring Batch**
 - **Spring Boot**
-- **H2 Database**
+- **Apache POI (Streaming API)**
+- **pjfanning**
+
+---
+
+## 🎯 Project Goals
 
----
+- Demonstrate **production-ready Spring Batch readers**
+- Show how to handle **large files efficiently**
+- Provide clean, extensible examples without framework overengineering

spring-batch-file-examples/pom.xml

@@ -28,6 +28,7 @@
         <spring.batch.excel.version>0.2.0</spring.batch.excel.version>
         <h2.version>2.4.240</h2.version>
         <jacoco.version>0.8.14</jacoco.version>
+        <excel.streaming.reader.version>5.2.0</excel.streaming.reader.version>
     </properties>
 
 	<dependencies>
@@ -69,20 +70,6 @@
             <version>${instancio.version}</version>
         </dependency>
 
-		<dependency>
-			<groupId>org.springframework.boot</groupId>
-			<artifactId>spring-boot-starter-test</artifactId>
-            <version>${spring.boot.version}</version>
-            <scope>test</scope>
-		</dependency>
-
-		<dependency>
-			<groupId>org.springframework.batch</groupId>
-			<artifactId>spring-batch-test</artifactId>
-			<scope>test</scope>
-            <version>${spring.batch.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.springframework.batch.extensions</groupId>
             <artifactId>spring-batch-excel</artifactId>
@@ -102,6 +89,26 @@
             <version>${poi.ooxml.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.pjfanning</groupId>
+            <artifactId>excel-streaming-reader</artifactId>
+            <version>${excel.streaming.reader.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <version>${spring.boot.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.batch</groupId>
+            <artifactId>spring-batch-test</artifactId>
+            <scope>test</scope>
+            <version>${spring.batch.version}</version>
+        </dependency>
+
     </dependencies>
 
 	<build>

spring-batch-file-examples/src/main/java/com/io/example/config/LargeExcelReadBatchConfig.java

@@ -0,0 +1,69 @@
+package com.io.example.config;
+
+import com.io.example.dto.StudentDto;
+import com.io.example.reader.StreamingExcelItemReader;
+import com.io.example.service.StudentService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.batch.item.ItemWriter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.time.LocalDate;
+
+@Configuration
+@RequiredArgsConstructor
+public class LargeExcelReadBatchConfig {
+
+    private final StudentService studentService;
+
+    @Bean
+    @StepScope
+    public StreamingExcelItemReader<StudentDto> largeExcelReader(
+            @Value("#{jobParameters['filePath']}") String filePath
+    ) {
+        return new StreamingExcelItemReader<>(
+                new ClassPathResource(filePath),
+                row -> new StudentDto(
+                        row.getCell(0).getStringCellValue(),
+                        row.getCell(1).getStringCellValue(),
+                        LocalDate.parse(row.getCell(2).getStringCellValue())
+                )
+        );
+    }
+
+    @Bean
+    public ItemProcessor<StudentDto, StudentDto> largeExcelProcessor() {
+        return student -> student;
+    }
+
+    @Bean
+    public ItemWriter<StudentDto> largeExcelWriter() {
+        return items -> items.forEach(studentService::print);
+    }
+
+    @Bean
+    public Step largeExcelStep(
+            JobRepository jobRepository,
+            PlatformTransactionManager transactionManager,
+            StreamingExcelItemReader<StudentDto> largeExcelReader,
+            ItemProcessor<StudentDto, StudentDto> largeExcelProcessor,
+            ItemWriter<StudentDto> largeExcelWriter,
+            @Value("${spring.batch.chunk-size}") int chunkSize
+    ) {
+        return new StepBuilder("largeExcelStep", jobRepository)
+                .<StudentDto, StudentDto>chunk(chunkSize, transactionManager)
+                .reader(largeExcelReader)
+                .processor(largeExcelProcessor)
+                .writer(largeExcelWriter)
+                .build();
+    }
+
+}

spring-batch-file-examples/src/main/java/com/io/example/config/SmallExcelReadBatchConfig.java

@@ -0,0 +1,63 @@
+package com.io.example.config;
+
+import com.io.example.dto.StudentDto;
+import com.io.example.mapper.StudentMapper;
+import com.io.example.service.StudentService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.extensions.excel.poi.PoiItemReader;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.batch.item.ItemWriter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.transaction.PlatformTransactionManager;
+
+@Configuration
+@RequiredArgsConstructor
+public class SmallExcelReadBatchConfig {
+
+    private final StudentService studentService;
+
+    @Bean
+    @StepScope
+    public PoiItemReader<StudentDto> smallExcelReader(
+            @Value("#{jobParameters['filePath']}") String filePath
+    ) {
+        PoiItemReader<StudentDto> reader = new PoiItemReader<>();
+        reader.setResource(new ClassPathResource(filePath));
+        reader.setLinesToSkip(1);
+        reader.setRowMapper(new StudentMapper());
+        return reader;
+    }
+
+    @Bean
+    public ItemProcessor<StudentDto, StudentDto> smallExcelProcessor() {
+        return student -> student;
+    }
+
+    @Bean
+    public ItemWriter<StudentDto> smallExcelWriter() {
+        return items -> items.forEach(studentService::print);
+    }
+
+    @Bean
+    public Step smallExcelStep(JobRepository jobRepository,
+                     PlatformTransactionManager transactionManager,
+                     PoiItemReader<StudentDto> smallExcelReader,
+                     ItemProcessor<StudentDto, StudentDto> smallExcelProcessor,
+                     ItemWriter<StudentDto> smallExcelWriter,
+                     @Value("${spring.batch.chunk-size}") int chunkSize) {
+
+        return new StepBuilder("smallExcelStep", jobRepository)
+                .<StudentDto, StudentDto>chunk(chunkSize, transactionManager)
+                .reader(smallExcelReader)
+                .processor(smallExcelProcessor)
+                .writer(smallExcelWriter)
+                .build();
+    }
+}