add new variable rawRecordDate to response and deprecate fileDate

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 2.6.2 [TODO]
+### Changed
+- Renamed `fileDate` to `rawRecordDate` in `response` documents to clarify that it represents the original raw record date.
+
+### Deprecated
+- Deprecated `fileDate` in `response` documents. It is still populated with the same value as `rawRecordDate` for backward compatibility.
+
 ## 2.6.1 [2026-04-29]
 ### Fixed
 - Fixed V1 schedules endpoint returning an empty list when V1 schedules have a null `partitionId`.

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -73,6 +73,7 @@ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit s
                 .mode(mode)
                 .recordDate(Instant.now())
                 .fileDate(su.getFileDate())
+                .rawRecordDate(su.getFileDate())
                 .build();
 
         return getCollectedDataFromSurveyUnit(su, surveyUnitModel, variablesMap, dataState);

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataParser.java

@@ -68,7 +68,9 @@ public LunaticXmlCampaign parseDataFile(Path filePath) throws GenesisException,
             if (surveyUnit.getNodeType() == Node.ELEMENT_NODE) {
                 Element surveyUnitElement = (Element) surveyUnit;
                 LunaticXmlSurveyUnit lunaticXmlSurveyUnit = new LunaticXmlSurveyUnit();
-                lunaticXmlSurveyUnit.setFileDate(getFileDate(filePath));
+                LocalDateTime rawRecordDate = getFileDate(filePath);
+                lunaticXmlSurveyUnit.setRawRecordDate(rawRecordDate);
+                lunaticXmlSurveyUnit.setFileDate(rawRecordDate);
                 lunaticXmlSurveyUnit.setId(surveyUnitElement.getElementsByTagName("Id").item(0).getFirstChild().getNodeValue());
                 lunaticXmlSurveyUnit.setQuestionnaireModelId(surveyUnitElement.getElementsByTagName("QuestionnaireModelId").item(0).getFirstChild().getNodeValue());
                 Node data = surveyUnitElement.getElementsByTagName("Data").item(0);

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataSequentialParser.java

@@ -24,12 +24,12 @@
  * It iterates through the file instead of storing the entire file into memory
  */
 public class LunaticXmlDataSequentialParser{
-    private final LocalDateTime fileDate;
+    private final LocalDateTime rawRecordDate;
     private final XMLEventReader reader;
 
 
      public LunaticXmlDataSequentialParser(final Path filePath, final InputStream stream) throws IOException, XMLStreamException {
-         this.fileDate = getFileDate(filePath);
+         this.rawRecordDate = getFileDate(filePath);
 
          XMLInputFactory factory = XMLInputFactory.newInstance();
          factory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
@@ -90,7 +90,8 @@ public LunaticXmlSurveyUnit readNextSurveyUnit() throws XMLStreamException {
      */
     private LunaticXmlSurveyUnit parseSurveyUnit(final XMLEventReader reader) throws XMLStreamException {
         LunaticXmlSurveyUnit xmlSurveyUnit = new LunaticXmlSurveyUnit();
-        xmlSurveyUnit.setFileDate(this.fileDate);
+        xmlSurveyUnit.setFileDate(this.rawRecordDate);
+        xmlSurveyUnit.setRawRecordDate(this.rawRecordDate);
 
         LunaticXmlData data = new LunaticXmlData();
 

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlSurveyUnit.java

@@ -11,6 +11,12 @@ public class LunaticXmlSurveyUnit {
 
     private String id;
     private String questionnaireModelId;
+    /**
+     * @deprecated use {@link #rawRecordDate} instead.
+     * This field is kept temporarily for backward compatibility and will be removed in a future version.
+     */
+    @Deprecated(since = "2026-05-11")
     private LocalDateTime fileDate;
+    private LocalDateTime rawRecordDate;
     private LunaticXmlData data;
 }

src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java

@@ -37,8 +37,15 @@ public class SurveyUnitModel {
 	private RawResponseDto.QuestionnaireStateEnum questionnaireState;
 	private LocalDateTime validationDate;
 	private Instant recordDate;
+    /**
+     * @deprecated use {@link #rawRecordDate} instead.
+     * This field is kept temporarily for backward compatibility and will be removed in a future version.
+     */
+    @Deprecated(since = "2026-05-11")
 	@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'hh:mm")
 	private LocalDateTime fileDate;
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'hh:mm")
+    private LocalDateTime rawRecordDate;
 
 	private List<VariableModel> collectedVariables;
 	private List<VariableModel> externalVariables;

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -299,6 +299,7 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
                         .isCapturedIndirectly(isCapturedIndirectly)
                         .state(dataState)
                         .fileDate(rawData.recordDate())
+                        .rawRecordDate(rawData.recordDate())
                         .recordDate(Instant.now())
                         .collectedVariables(new ArrayList<>())
                         .externalVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java

@@ -270,6 +270,7 @@ public List<SurveyUnitModel> convertRawResponse(List<RawResponseModel> rawRespon
                         .isCapturedIndirectly(isCapturedIndirectly)
                         .state(dataState)
                         .fileDate(rawResponseModel.recordDate())
+                        .rawRecordDate(rawResponseModel.recordDate())
                         .recordDate(Instant.now())
                         .collectedVariables(new ArrayList<>())
                         .externalVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java

@@ -80,6 +80,7 @@ private static SurveyUnitModel createFormattedSurveyUnitModel(
                 .mode(sampleSurveyUnitModel.getMode())
                 .recordDate(Instant.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED
                 .fileDate(sampleSurveyUnitModel.getFileDate())
+                .rawRecordDate(sampleSurveyUnitModel.getRawRecordDate())
                 .collectedVariables(new ArrayList<>())
                 .externalVariables(new ArrayList<>())
                 .build();

src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java

@@ -1,5 +1,6 @@
 package fr.insee.genesis.infrastructure.document.surveyunit;
 
+import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import fr.insee.genesis.Constants;
 import fr.insee.modelefiliere.RawResponseDto;
@@ -46,7 +47,13 @@ public class SurveyUnitDocument {
 	@Indexed
 	private String mode;
 	private Instant recordDate;
+    /**
+     * @deprecated use {@link #rawRecordDate} instead.
+     * This field is kept temporarily for backward compatibility and will be removed in a future version.
+     */
+    @Deprecated(since = "2026-05-11")
 	private LocalDateTime fileDate;
+    private LocalDateTime rawRecordDate;
 	private List<VariableDocument> collectedVariables;
 	private List<VariableDocument> externalVariables;
 	private String modifiedBy;

src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java

@@ -250,18 +250,20 @@ void convert_shouldSetRecordDateCloseToNow() {
         }
 
         @Test
-        @DisplayName("Should map fileDate from survey unit")
-        void convert_shouldMapFileDate() {
+        @DisplayName("Should map rawRecordDate from survey unit")
+        void convert_shouldMapRawRecordDate() {
             // GIVEN
-            LocalDateTime fileDate = LocalDateTime.of(2024, 1, 15, 10, 0);
+            LocalDateTime rawRecordDate = LocalDateTime.of(2024, 1, 15, 10, 0);
             LunaticXmlSurveyUnit su = buildSurveyUnit(List.of(collectedDataWithValue("VAR1", "val")));
-            su.setFileDate(fileDate);
+            su.setFileDate(rawRecordDate);
+            su.setRawRecordDate(rawRecordDate);
 
             // WHEN
             SurveyUnitModel collected = getCollected(LunaticXmlAdapter.convert(su, VARIABLES_MAP, MODE));
 
             // THEN
-            assertThat(collected.getFileDate()).isEqualTo(fileDate);
+            assertThat(collected.getFileDate()).isEqualTo(rawRecordDate);
+            assertThat(collected.getRawRecordDate()).isEqualTo(rawRecordDate);
         }
     }
 

src/test/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModelTest.java

@@ -56,6 +56,7 @@ void noArgsConstructor_shouldCreateEmptyModel() {
             assertThat(model.getValidationDate()).isNull();
             assertThat(model.getRecordDate()).isNull();
             assertThat(model.getFileDate()).isNull();
+            assertThat(model.getRawRecordDate()).isNull();
             assertThat(model.getCollectedVariables()).isNull();
             assertThat(model.getExternalVariables()).isNull();
             assertThat(model.getModifiedBy()).isNull();
@@ -94,6 +95,7 @@ void allArgsConstructor_shouldSetAllFields() {
                     now,
                     now.toInstant(ZoneOffset.UTC),
                     now,
+                    now,
                     collected,
                     external,
                     MODIFIED_BY
@@ -111,6 +113,7 @@ void allArgsConstructor_shouldSetAllFields() {
             assertThat(model.getValidationDate()).isEqualTo(now);
             assertThat(model.getRecordDate()).isEqualTo(now.toInstant(ZoneOffset.UTC));
             assertThat(model.getFileDate()).isEqualTo(now);
+            assertThat(model.getRawRecordDate()).isEqualTo(now);
             assertThat(model.getCollectedVariables()).isEqualTo(collected);
             assertThat(model.getExternalVariables()).isEqualTo(external);
             assertThat(model.getModifiedBy()).isEqualTo(MODIFIED_BY);
@@ -144,6 +147,7 @@ void builder_shouldSetAllFields() {
                     .validationDate(now)
                     .recordDate(now.toInstant(ZoneOffset.UTC))
                     .fileDate(now)
+                    .rawRecordDate(now)
                     .collectedVariables(collected)
                     .externalVariables(external)
                     .modifiedBy(MODIFIED_BY)
@@ -161,6 +165,7 @@ void builder_shouldSetAllFields() {
             assertThat(model.getValidationDate()).isEqualTo(now);
             assertThat(model.getRecordDate()).isEqualTo(now.toInstant(ZoneOffset.UTC));
             assertThat(model.getFileDate()).isEqualTo(now);
+            assertThat(model.getRawRecordDate()).isEqualTo(now);
             assertThat(model.getCollectedVariables()).isEqualTo(collected);
             assertThat(model.getExternalVariables()).isEqualTo(external);
             assertThat(model.getModifiedBy()).isEqualTo(MODIFIED_BY);

src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java

@@ -50,7 +50,9 @@ static void init(){
         surveyUnitDocumentStatic.setUsualSurveyUnitId(USUAL_SURVEY_UNIT_ID);
         surveyUnitDocumentStatic.setInterrogationId(INTERROGATION_ID);
         surveyUnitDocumentStatic.setState("COLLECTED");
-        surveyUnitDocumentStatic.setFileDate(LocalDateTime.of(2023,1,1,0,0,0));
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023, 1, 1, 0, 0, 0);
+        surveyUnitDocumentStatic.setFileDate(rawRecordDate);
+        surveyUnitDocumentStatic.setRawRecordDate(rawRecordDate);
 
         List<VariableDocument> documentExternalVariableList = new ArrayList<>();
         VariableDocument externalVariable = new VariableDocument();
@@ -88,7 +90,8 @@ static void init(){
         deprecatedSurveyUnitDocumentStatic.setIdUE(ID_UE);
         deprecatedSurveyUnitDocumentStatic.setInterrogationId(INTERROGATION_ID);
         deprecatedSurveyUnitDocumentStatic.setState("COLLECTED");
-        deprecatedSurveyUnitDocumentStatic.setFileDate(LocalDateTime.of(2023,1,1,0,0,0));
+        deprecatedSurveyUnitDocumentStatic.setFileDate(rawRecordDate);
+        deprecatedSurveyUnitDocumentStatic.setRawRecordDate(rawRecordDate);
 
         documentExternalVariableList = new ArrayList<>();
         externalVariable = new VariableDocument();
@@ -125,7 +128,8 @@ static void init(){
                 .interrogationId(INTERROGATION_ID)
                 .collectionInstrumentId(COLLECTION_INSTRUMENT_ID)
                 .state(DataState.COLLECTED)
-                .fileDate(LocalDateTime.of(2023,1,1,0,0,0))
+                .fileDate(rawRecordDate)
+                .rawRecordDate(rawRecordDate)
                 .recordDate(LocalDateTime.of(2024,1,1,0,0,0).toInstant(ZoneOffset.UTC))
                 .externalVariables(externalVariableModelList)
                 .collectedVariables(collectedVariableList)
@@ -147,13 +151,15 @@ void shouldReturnNull(){
     @DisplayName("Should convert survey unit document to model")
     void shouldReturnModelFromDocument(){
         SurveyUnitModel surveyUnit = surveyUnitDocumentMapperImplStatic.documentToModel(surveyUnitDocumentStatic);
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023,1,1,0,0,0);
 
         Assertions.assertThat(surveyUnit.getMode()).isEqualTo(Mode.WEB);
         Assertions.assertThat(surveyUnit.getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnit.getUsualSurveyUnitId()).isEqualTo(USUAL_SURVEY_UNIT_ID);
         Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0));
+        Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(rawRecordDate);
+        Assertions.assertThat(surveyUnit.getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnit.getExternalVariables()).filteredOn(externalVariableModel ->
             externalVariableModel.varId().equals(VAR_ID)
@@ -171,13 +177,15 @@ void shouldReturnModelFromDocument(){
     @DisplayName("Should convert deprecated survey unit document to model")
     void shouldReturnModelFromDeprecatedDocument(){
         SurveyUnitModel surveyUnit = surveyUnitDocumentMapperImplStatic.documentToModel(deprecatedSurveyUnitDocumentStatic);
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023,1,1,0,0,0);
 
         Assertions.assertThat(surveyUnit.getMode()).isEqualTo(Mode.WEB);
         Assertions.assertThat(surveyUnit.getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo(QUESTIONNAIRE_ID);
         Assertions.assertThat(surveyUnit.getUsualSurveyUnitId()).isEqualTo(ID_UE);
         Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0));
+        Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(rawRecordDate);
+        Assertions.assertThat(surveyUnit.getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnit.getExternalVariables()).filteredOn(externalVariableModel ->
                 externalVariableModel.varId().equals(VAR_ID)
@@ -195,12 +203,14 @@ void shouldReturnModelFromDeprecatedDocument(){
     @DisplayName("Should convert survey unit model to document")
     void shouldReturnDocumentFromModel(){
         SurveyUnitDocument surveyUnitDocument = surveyUnitDocumentMapperImplStatic.modelToDocument(surveyUnitStatic);
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023,1,1,0,0,0);
 
         Assertions.assertThat(surveyUnitDocument.getMode()).isEqualTo(MODE);
         Assertions.assertThat(surveyUnitDocument.getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnitDocument.getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnitDocument.getState()).isEqualTo("COLLECTED");
-        Assertions.assertThat(surveyUnitDocument.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0));
+        Assertions.assertThat(surveyUnitDocument.getFileDate()).isEqualTo(rawRecordDate);
+        Assertions.assertThat(surveyUnitDocument.getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitDocument.getExternalVariables()).filteredOn(externalVariableDocument ->
                 externalVariableDocument.getVarId().equals(VAR_ID)
@@ -222,13 +232,15 @@ void shouldReturnModelListFromDocumentList(){
         surveyUnitDocumentList.add(surveyUnitDocumentStatic);
 
         List<SurveyUnitModel> surveyUnitList = surveyUnitDocumentMapperImplStatic.listDocumentToListModel(surveyUnitDocumentList);
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023,1,1,0,0,0);
 
         Assertions.assertThat(surveyUnitList.getFirst().getMode()).isEqualTo(Mode.WEB);
         Assertions.assertThat(surveyUnitList.getFirst().getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getUsualSurveyUnitId()).isEqualTo(USUAL_SURVEY_UNIT_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0));
+        Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
+        Assertions.assertThat(surveyUnitList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitList.getFirst().getExternalVariables()).filteredOn(externalVariableModel ->
                 externalVariableModel.varId().equals(VAR_ID)
@@ -248,13 +260,15 @@ void shouldReturnModelListFromDeprecatedDocumentList(){
         surveyUnitDocumentList.add(deprecatedSurveyUnitDocumentStatic);
 
         List<SurveyUnitModel> surveyUnitList = surveyUnitDocumentMapperImplStatic.listDocumentToListModel(surveyUnitDocumentList);
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023,1,1,0,0,0);
 
         Assertions.assertThat(surveyUnitList.getFirst().getMode()).isEqualTo(Mode.WEB);
         Assertions.assertThat(surveyUnitList.getFirst().getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo(QUESTIONNAIRE_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getUsualSurveyUnitId()).isEqualTo(ID_UE);
         Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0));
+        Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
+        Assertions.assertThat(surveyUnitList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitList.getFirst().getExternalVariables()).filteredOn(externalVariableModel ->
                 externalVariableModel.varId().equals(VAR_ID)
@@ -274,12 +288,13 @@ void shouldReturnDocumentListFromModelList(){
         surveyUnitList.add(surveyUnitStatic);
 
         List<SurveyUnitDocument> surveyUnitDocumentList = surveyUnitDocumentMapperImplStatic.listModelToListDocument(surveyUnitList);
-
+        LocalDateTime rawRecordDate = LocalDateTime.of(2023,1,1,0,0,0);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getMode()).isEqualTo(MODE);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getState()).isEqualTo("COLLECTED");
-        Assertions.assertThat(surveyUnitDocumentList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0));
+        Assertions.assertThat(surveyUnitDocumentList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
+        Assertions.assertThat(surveyUnitDocumentList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getExternalVariables()).filteredOn(externalVariableDocument ->
                 externalVariableDocument.getVarId().equals(VAR_ID)