Update testing

Author: rweinsteMW

.github/workflows/ci.yml

@@ -13,29 +13,23 @@ jobs:
     strategy: 
       fail-fast: false
       matrix:
-        MATLABVersion: [R2023a,R2023b]
+        MATLABVersion: [R2021a,R2021b,R2022a,R2022b,R2023a,R2023b]
     runs-on: ubuntu-latest
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      # Checks-out your repository
       - uses: actions/checkout@v3
 
-      # Sets up MATLAB on the GitHub Actions runner
+      # Sets up MATLAB
       - name: Setup MATLAB
         uses: matlab-actions/setup-matlab@v1
         with:
           release: ${{ matrix.MATLABVersion }}
 
-      # Run SmokeTests
+      # Run all the tests
       - name: Run SmokeTests
         uses: matlab-actions/run-command@v1
         with:
-          command: openProject(pwd); results = runtests(fullfile("SoftwareTests","SmokeTests.m")); assertSuccess(results); 
-     
-      # Run FunctionTests
-      - name: Run FunctionTests
-        uses: matlab-actions/run-command@v1
-        with:
-          command: openProject(pwd); results = runtests(fullfile("SoftwareTests","FunctionTests.m")); assertSuccess(results); 
+          command: openProject(pwd); RunAllTests; 
 
       # Upload the test results as artifact
       - name: Upload TestResults
@@ -44,6 +38,23 @@ jobs:
           name: TestResults
           path: ./SoftwareTests/TestResults_${{ matrix.MATLABVersion }}.txt
 
+  badge:
+    if: ${{ always() }}
+    needs: [test]
+    strategy: 
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps: 
+
+      # Checks-out your repository
+      - uses: actions/checkout@v3
+      
+      # Sets up R2023b
+      - name: Setup MATLAB
+        uses: matlab-actions/setup-matlab@v1
+        with:
+          release: R2023b
+  
       # Download the test results from artifact
       - name: Download TestResults
         uses: actions/download-artifact@v2.1.1
@@ -55,14 +66,16 @@ jobs:
       - name: Run CreateBadge
         uses: matlab-actions/run-command@v1
         with:
-          command: openProject(pwd); results = runtests(fullfile("SoftwareTests","CreateBadge.m"));
+          command: openProject(pwd); CreateBadge;
 
       # Commit the JSON for the MATLAB releases badge 
       - name: Commit changed files
         continue-on-error: true
         run: |
           git config user.name "${{ github.workflow }} by ${{ github.actor }}"
           git config user.email "<>"
+          git pull
+          git add Images/TestedWith.json
           git commit Images/TestedWith.json -m "Update CI badges ${{  github.ref_name }}"
           git fetch
           git push

.gitignore

@@ -49,3 +49,6 @@ Utilities/ProjectSettings.mat
 
 # Test results
 SoftwareTests/TestResults_*
+
+# GitLab page fodler
+public/

Scripts/FE1_ProgrammaticML.mlx

@@ -0,0 +1,47 @@
+classdef CheckTestResults < matlab.unittest.TestCase
+
+    properties (SetAccess = protected)
+    end
+
+    properties (ClassSetupParameter)
+        Project = {''};
+    end
+
+    properties (TestParameter)
+        Version
+    end
+
+
+    methods (TestParameterDefinition,Static)
+
+        function Version = GetResults(Project)
+            RootFolder = currentProject().RootFolder;
+            Version = dir(fullfile(RootFolder,"SoftwareTests","TestResults*.txt"));
+            Version = extractBetween([Version.name],"TestResults_",".txt");
+        end
+
+    end
+
+    methods (TestClassSetup)
+
+        function SetUpSmokeTest(testCase,Project)
+            try
+               currentProject;   
+            catch
+                error("Project is not loaded.")
+            end
+        end
+
+    end
+
+    methods(Test)
+
+        function CheckResults(testCase,Version)
+            File = fullfile("SoftwareTests","TestResults_"+Version+".txt");
+            Results = readtable(File,TextType="string");
+            testCase.verifyTrue(all(Results.Passed));
+        end
+
+    end
+
+end

Scripts/FE2_LoadForecastDL.mlx

@@ -1,82 +1,28 @@
-% Run these tests with runMyTests
-% All tests so far are on code expected to run without errors
-% If/when we end up with a version that _should_ error, 
-% please add it to this set of examples
-classdef CreateBadge < matlab.unittest.TestCase
-    
-    properties
-        rootProject
-        results
-    end
-
-
-     methods (TestClassSetup)
-
-        function setUpPath(testCase)
-            
-            try
-                project = currentProject;
-                testCase.rootProject = project.RootFolder;
-                cd(testCase.rootProject)
-            catch
-                error("Load project prior to run tests")
-            end
-            
-            testCase.log("Running in " + version)
-
-        end % function setUpPath
-
-        function readResults(testCase)
-            Release = string([]);
-            Passed = [];
-            testCase.results = table(Release,Passed);
-
-            ResultFiles = dir("SoftwareTests"+filesep+"TestResults_*");
-            for kFiles = 1:size(ResultFiles)
-                Results = readtable(fullfile(ResultFiles(kFiles).folder,ResultFiles(kFiles).name),...
-                    Delimiter=",",TextType="string");
-                Release = Results.Version(1);
-                Passed = all(Results.Status == "passed");
-                testCase.results(end+1,:) = table(Release,Passed);
-            end
-        end
-
-    end % methods (TestClassSetup)
-
-    methods(Test)
-
-        function writeBadge(testCase)
-            
-            % Create JSON
-            badgeInfo = struct;
-            badgeInfo.schemaVersion = 1;
-            badgeInfo.label = "tested with";
-            badgeInfo.message = "";
-            
-            % Check that results exist:
-            if size(testCase.results,1) == 0
-                badgeInfo.message = "None";
-                badgeInfo.color = "failed";
-            else
-                for i = 1:size(testCase.results,1)
-                    if testCase.results.Passed(i)
-                        if badgeInfo.message ~= ""
-                            badgeInfo.message = badgeInfo.message + " | ";
-                        end
-                        badgeInfo.message = badgeInfo.message + string(testCase.results.Release(i));
-                    end
-                end
-                badgeInfo.color = "success";
-            end
-
-            % Write JSON file out
-            badgeJSON = jsonencode(badgeInfo);
-            fid = fopen(fullfile("Images","TestedWith.json"),"w");
-            fwrite(fid,badgeJSON);
-            fclose(fid);
-            
-        end
-
-    end
-
-end
+% Create the test suite with SmokeTest and Function test if they exist
+Suite = testsuite("CheckTestResults");
+
+% Create a runner with no plugins
+Runner = matlab.unittest.TestRunner.withNoPlugins;
+
+% Run the test suite
+Results = Runner.run(Suite);
+
+% Format the results in a table and save them
+Results = table(Results');
+Results = Results(Results.Passed,:);
+Version = extractBetween(string(Results.Name),"Version=",")");
+
+
+% Format the JSON file
+Badge = struct;
+Badge.schemaVersion = 1;
+Badge.label = "Tested with";
+if size(Results,1) >= 1
+    Badge.color = "success"
+    Badge.message = join(Version," | ");
+else
+    Badge.color = "failure";
+    Badge.message = "Pipeline fails";
+end
+Badge = jsonencode(Badge);
+writelines(Badge,fullfile("Images","TestedWith.json"));

Scripts/LoadForecastRegression.mlx

@@ -0,0 +1,48 @@
+function RunAllTest(EnableReport,ReportFolder)
+arguments
+    EnableReport (1,1) logical = false;
+    ReportFolder (1,1) string = "public";
+end
+
+import matlab.unittest.plugins.TestReportPlugin;
+
+% Create a runner
+Runner = matlab.unittest.TestRunner.withTextOutput;
+if EnableReport
+    Folder = fullfile(currentProject().RootFolder,ReportFolder);
+    if ~isfolder(Folder)
+        mkdir(Folder)
+    else
+        rmdir(Folder,'s')
+        mkdir(Folder)
+    end
+    Plugin = TestReportPlugin.producingHTML(Folder,...
+        "IncludingPassingDiagnostics",true,...
+        "IncludingCommandWindowText",true,...
+        "LoggingLevel",matlab.automation.Verbosity(1));
+        Runner.addPlugin(Plugin);
+end
+
+% Create the test suite with SmokeTest and Function test if they exist
+Suite = testsuite("SmokeTests");
+Suite = [Suite testsuite("FunctionTests")];
+
+% Run the test suite
+Results = Runner.run(Suite);
+
+if EnableReport
+    web(fullfile(Folder,"index.html"))
+else
+    T = table(Results);
+    disp(newline + "Test summary:")
+    disp(T)
+end
+
+% Format the results in a table and save them
+ResultsTable = table(Results')
+writetable(ResultsTable,fullfile("SoftwareTests","TestResults_"+release_version+".txt"));
+
+% Assert success of test
+assertSuccess(Results);
+
+end