Add DataProvider execution support to Allure TestNG setup section

allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java

@@ -39,6 +39,8 @@
 import org.testng.IAttributes;
 import org.testng.IClass;
 import org.testng.IConfigurationListener;
+import org.testng.IDataProviderListener;
+import org.testng.IDataProviderMethod;
 import org.testng.IInvokedMethod;
 import org.testng.IInvokedMethodListener;
 import org.testng.IMethodInstance;
@@ -110,7 +112,8 @@ public class AllureTestNg implements
         ITestListener,
         IInvokedMethodListener,
         IConfigurationListener,
-        IMethodInterceptor {
+        IMethodInterceptor,
+        IDataProviderListener {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(AllureTestNg.class);
 
@@ -150,6 +153,10 @@ public class AllureTestNg implements
      * Store uuid for class test containers.
      */
     private final Map<ITestClass, String> classContainerUuidStorage = new ConcurrentHashMap<>();
+    /**
+     * Store uuid for data provider containers.
+     */
+    private final Map<ITestNGMethod, String> dataProviderContainerUuidStorage = new ConcurrentHashMap<>();
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
     private final AllureLifecycle lifecycle;
     private final AllureTestNgTestFilter testFilter;
@@ -280,6 +287,14 @@ public void onAfterClass(final ITestClass testClass) {
             getLifecycle().stopTestContainer(uuid);
             getLifecycle().writeTestContainer(uuid);
         });
+        dataProviderContainerUuidStorage.entrySet().removeIf(entry -> {
+            if (entry.getKey().getTestClass().equals(testClass)) {
+                getLifecycle().stopTestContainer(entry.getValue());
+                getLifecycle().writeTestContainer(entry.getValue());
+                return true;
+            }
+            return false;
+        });
     }
 
     @Override
@@ -302,6 +317,10 @@ public void onTestStart(final ITestResult testResult) {
                 .map(ITestResult::getMethod)
                 .map(ITestNGMethod::getTestClass)
                 .ifPresent(clazz -> addClassContainerChild(clazz, uuid));
+
+        Optional.of(testResult)
+                .map(ITestResult::getMethod)
+                .ifPresent(method -> addDataProviderContainerChild(method, uuid));
     }
 
     @SuppressWarnings("BooleanExpressionComplexity")
@@ -626,6 +645,65 @@ public void onConfigurationSkip(final ITestResult itr) {
         //do nothing
     }
 
+    @Override
+    public void beforeDataProviderExecution(final IDataProviderMethod dataProviderMethod,
+                                            final ITestNGMethod method,
+                                            final ITestContext iTestContext) {
+        currentExecutable.remove();
+        final String containerUuid = dataProviderContainerUuidStorage.computeIfAbsent(
+                method,
+                key -> {
+                    final String uuid = UUID.randomUUID().toString();
+                    getLifecycle().startTestContainer(
+                            new TestResultContainer()
+                                    .setUuid(uuid)
+                                    .setName(method.getMethodName())
+                    );
+                    return uuid;
+                }
+        );
+
+        final String uuid = currentExecutable.get();
+        final FixtureResult result = new FixtureResult()
+                .setName(dataProviderMethod.getMethod().getName())
+                .setStage(Stage.RUNNING);
+
+        processDescription(
+                getClass().getClassLoader(),
+                dataProviderMethod.getMethod(),
+                result::setDescription,
+                result::setDescriptionHtml
+        );
+
+        getLifecycle().startPrepareFixture(containerUuid, uuid, result);
+    }
+
+    @Override
+    public void afterDataProviderExecution(final IDataProviderMethod dataProviderMethod,
+                                           final ITestNGMethod method,
+                                           final ITestContext iTestContext) {
+        final String uuid = currentExecutable.get();
+        getLifecycle().updateFixture(uuid, result -> {
+            if (result.getStatus() == null) {
+                result.setStatus(Status.PASSED);
+            }
+        });
+        getLifecycle().stopFixture(uuid);
+        currentExecutable.remove();
+    }
+
+    @Override
+    public void onDataProviderFailure(final ITestNGMethod method,
+                                      final ITestContext ctx,
+                                      final RuntimeException t) {
+        final String uuid = currentExecutable.get();
+        getLifecycle().updateFixture(uuid, result -> result
+                .setStatus(getStatus(t))
+                .setStatusDetails(getStatusDetails(t).orElse(null)));
+        getLifecycle().stopFixture(uuid);
+        currentExecutable.remove();
+    }
+
     protected String getHistoryId(final ITestNGMethod method, final List<Parameter> parameters) {
         final MessageDigest digest = getMd5Digest();
         final String testClassName = method.getTestClass().getName();
@@ -827,6 +905,10 @@ private Consumer<TestResult> setStatus(final Status status, final StatusDetails
         };
     }
 
+    private void addDataProviderContainerChild(final ITestNGMethod method, final String childUuid) {
+        this.addChildToContainer(dataProviderContainerUuidStorage.get(method), childUuid);
+    }
+
     private void addClassContainerChild(final ITestClass clazz, final String childUuid) {
         this.addChildToContainer(classContainerUuidStorage.get(clazz), childUuid);
     }

allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java

@@ -507,7 +507,7 @@ public void multipleSuites() {
                 .hasSize(3);
         List<String> uids = testResults.stream().map(TestResult::getUuid).collect(Collectors.toList());
         assertThat(testContainers).as("Unexpected quantity of testng containers has been written")
-                .hasSize(8).extracting(TestResultContainer::getName)
+                .hasSize(9).extracting(TestResultContainer::getName)
                 .contains(beforeMethodName, beforeMethodName, firstTagName, firstSuiteName, secondTagName,
                         secondSuiteName);
 
@@ -553,7 +553,7 @@ public void parallelMethods() {
         assertThat(testResults).as("Unexpected quantity of testng case results has been written")
                 .hasSize(2001);
         assertThat(testContainers).as("Unexpected quantity of testng containers has been written")
-                .hasSize(6006);
+                .hasSize(6007);
 
         assertContainersPerMethod(before1, testContainers, uids);
         assertContainersPerMethod(before2, testContainers, uids);
@@ -1515,6 +1515,50 @@ public AllureResults runTestPlan(final TestPlan plan, final Class<?>... testClas
         });
     }
 
+    @AllureFeatures.Fixtures
+    @Test(description = "Should process data provider in setup")
+    public void shouldProcessDataProviderInSetup() {
+        final AllureResults results = runTestNgSuites("suites/data-provider-with-attachment.xml");
+
+        assertThat(results.getTestResultContainers())
+                .flatExtracting(TestResultContainer::getBefores)
+                .extracting(FixtureResult::getName, FixtureResult::getStatus)
+                .contains(Tuple.tuple("dataProvider", Status.PASSED));
+
+        assertThat(results.getTestResultContainers())
+                .flatExtracting(TestResultContainer::getBefores)
+                .filteredOn("name", "dataProvider")
+                .flatExtracting(FixtureResult::getAttachments)
+                .hasSize(1)
+                .extracting(Attachment::getName)
+                .contains("attachment");
+    }
+
+    @AllureFeatures.Fixtures
+    @Test(description = "Should process failed data provider in setup")
+    public void shouldProcessFailedDataProviderInSetup() {
+        final AllureResults results = runTestNgSuites("suites/failed-data-provider.xml");
+
+        assertThat(results.getTestResultContainers())
+                .flatExtracting(TestResultContainer::getBefores)
+                .extracting(FixtureResult::getName, FixtureResult::getStatus)
+                .contains(Tuple.tuple("dataProvider", Status.BROKEN));
+    }
+
+    @AllureFeatures.Fixtures
+    @Test(description = "Should process flaky data provider in setup")
+    public void shouldProcessFlakyDataProvider() {
+        final AllureResults results = runTestNgSuites("suites/flaky-data-provider.xml");
+
+        assertThat(results.getTestResultContainers())
+                .flatExtracting(TestResultContainer::getBefores)
+                .extracting(FixtureResult::getName, FixtureResult::getStatus)
+                .containsSubsequence(
+                        Tuple.tuple("provide", Status.BROKEN),
+                        Tuple.tuple("provide", Status.PASSED)
+                );
+    }
+
     private Integer getOrderParameter(final TestResult result) {
         return result.getParameters().stream()
                 .filter(p -> p.getName().equals("order"))

allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java

@@ -0,0 +1,35 @@
+/*
+ *  Copyright 2016-2024 Qameta Software Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package io.qameta.allure.testng.samples;
+
+import io.qameta.allure.Allure;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class DataProviderWithAttachment {
+
+    @DataProvider
+    public Object[][] dataProvider() {
+        Allure.addAttachment("attachment", "attachment content");
+        return new Object[][]{
+            {"a"}
+        };
+    }
+
+    @Test(dataProvider = "dataProvider")
+    public void test(String s) {
+    }
+}

allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java

@@ -0,0 +1,31 @@
+/*
+ *  Copyright 2016-2024 Qameta Software Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package io.qameta.allure.testng.samples;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class FailedDataProvider {
+
+    @DataProvider
+    public Object[][] dataProvider() {
+        throw new RuntimeException("Data provider failed");
+    }
+
+    @Test(dataProvider = "dataProvider")
+    public void test(String s) {
+    }
+}

allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java

@@ -0,0 +1,53 @@
+/*
+ *  Copyright 2016-2024 Qameta Software Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package io.qameta.allure.testng.samples;
+
+import org.testng.IDataProviderMethod;
+import org.testng.IRetryDataProvider;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class FlakyDataProvider {
+
+    private static final AtomicInteger COUNTER = new AtomicInteger(0);
+
+    @BeforeClass
+    public void reset() {
+        COUNTER.set(0);
+    }
+
+    public static class Retry implements IRetryDataProvider {
+        @Override
+        public boolean retry(IDataProviderMethod method) {
+            return COUNTER.incrementAndGet() < 2;
+        }
+    }
+
+    @DataProvider(retryUsing = Retry.class)
+    public Object[][] provide() {
+        if (COUNTER.get() == 0) {
+            throw new RuntimeException("Simulated DataProvider Failure");
+        }
+        return new Object[][]{{"data"}};
+    }
+
+    @Test(dataProvider = "provide")
+    public void test(String s) {
+    }
+}

allure-testng/src/test/resources/suites/data-provider-with-attachment.xml

@@ -0,0 +1,8 @@
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
+<suite name="Data Provider Suite" verbose="1" parallel="false" thread-count="1">
+    <test name="Data Provider Test">
+        <classes>
+            <class name="io.qameta.allure.testng.samples.DataProviderWithAttachment" />
+        </classes>
+    </test>
+</suite>

allure-testng/src/test/resources/suites/failed-data-provider.xml

@@ -0,0 +1,8 @@
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
+<suite name="Failed Data Provider Suite" verbose="1" parallel="false" thread-count="1">
+    <test name="Failed Data Provider Test">
+        <classes>
+            <class name="io.qameta.allure.testng.samples.FailedDataProvider" />
+        </classes>
+    </test>
+</suite>

allure-testng/src/test/resources/suites/flaky-data-provider.xml

@@ -0,0 +1,8 @@
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
+<suite name="Flaky Data Provider Suite" verbose="1" parallel="false" thread-count="1">
+    <test name="Flaky Data Provider Test">
+        <classes>
+            <class name="io.qameta.allure.testng.samples.FlakyDataProvider" />
+        </classes>
+    </test>
+</suite>