SimX-SST Integration Changes (No Memory Integration)

.github/workflows/ci.yml

@@ -59,12 +59,20 @@ jobs:
           cd build
           ../configure --tooldir=$TOOLDIR
           ci/toolchain_install.sh --all
+          ci/sst_install.sh
 
       - name: Setup Third Party
         if: steps.cache-thirdparty.outputs.cache-hit != 'true'
         run: |
           make -C third_party > /dev/null
 
+      - name: Export SST paths
+        run: |
+          echo "$PWD/tools/sst-install/sst-core/bin" >> $GITHUB_PATH
+          echo "$PWD/tools/sst-install/sst-elements/bin" >> $GITHUB_PATH
+          echo "SST_CORE_HOME=$PWD/tools/sst-install/sst-core" >> $GITHUB_ENV
+          echo "SST_ELEMENTS_HOME=$PWD/tools/sst-install/sst-elements" >> $GITHUB_ENV
+
   build:
     needs: setup
     strategy:
@@ -99,6 +107,13 @@ jobs:
           restore-keys: |
             ${{ matrix.os }}-thirdparty-
 
+      - name: Export SST paths
+        run: |
+          echo "$PWD/tools/sst-install/sst-core/bin" >> $GITHUB_PATH
+          echo "$PWD/tools/sst-install/sst-elements/bin" >> $GITHUB_PATH
+          echo "SST_CORE_HOME=$PWD/tools/sst-install/sst-core" >> $GITHUB_ENV
+          echo "SST_ELEMENTS_HOME=$PWD/tools/sst-install/sst-elements" >> $GITHUB_ENV
+
       - name: Run Build
         run: |
           TOOLDIR=$PWD/tools
@@ -157,6 +172,13 @@ jobs:
           name: build-${{ matrix.os }}-${{ matrix.xlen }}
           path: build${{ matrix.xlen }}
 
+      - name: Export SST paths
+        run: |
+          echo "$PWD/tools/sst-install/sst-core/bin" >> $GITHUB_PATH
+          echo "$PWD/tools/sst-install/sst-elements/bin" >> $GITHUB_PATH
+          echo "SST_CORE_HOME=$PWD/tools/sst-install/sst-core" >> $GITHUB_ENV
+          echo "SST_ELEMENTS_HOME=$PWD/tools/sst-install/sst-elements" >> $GITHUB_ENV
+
       - name: Run tests
         run: |
           cd build${{ matrix.xlen }}
@@ -167,6 +189,7 @@ jobs:
             ./ci/regression.sh --isa
             ./ci/regression.sh --kernel
             ./ci/regression.sh --regression
+            ./ci/regression.sh --sst_tests
           else
             ./ci/regression.sh --${{ matrix.name }}
           fi

ci/regression.sh.in

@@ -78,6 +78,23 @@ kernel()
     echo "kernel tests done!"
 }
 
+sst_tests()
+{
+    echo "begin sst tests..."
+
+    make -C sim/simx USE_SST=1
+    make -C tests/kernel
+
+    cp sim/simx/libvortex.so $SST_ELEMENTS_HOME/lib/sst-elements-library/   # alternatively - $ sst --add-lib-path `pwd` myConfig.py
+
+    sst ci/sst_test_vortex_hello.py
+    sst ci/sst_test_vortex_fibonacci.py
+    sst ci/sst_test_vortex_vecadd.py
+    sst ci/sst_test_vortex_conform.py
+
+    echo "sst tests done!"
+}
+
 regression()
 {
     echo "begin regression tests..."
@@ -505,7 +522,7 @@ cupbop() {
 show_usage()
 {
     echo "Vortex Regression Test"
-    echo "Usage: $0 [--clean] [--unittest] [--isa] [--kernel] [--regression] [--opencl] [--cache] [--config1] [--config2] [--debug] [--scope] [--stress] [--synthesis] [--vector] [--tensor] [--cupbop] [--all] [--h|--help]"
+    echo "Usage: $0 [--clean] [--unittest] [--isa] [--kernel] [--regression] [--opencl] [--cache] [--config1] [--config2] [--debug] [--scope] [--stress] [--synthesis] [--vector] [--tensor] [--cupbop] [--sst_tests] [--all] [--h|--help]"
 }
 
 declare -a tests=()
@@ -564,6 +581,9 @@ while [ "$1" != "" ]; do
         --cupbop )
                 tests+=("cupbop")
                 ;;
+        --sst_tests )
+                tests+=("sst_tests")
+                ;;
         --all )
                 tests=()
                 tests+=("unittest")

ci/sst_install.sh.in

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# Copyright © 2019-2023
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# exit when any command fails
+set -e
+
+OPENMPI_416=https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.gz  # version dependency
+SST_CORE_1510=https://github.com/sstsimulator/sst-core/releases/download/v15.1.0_Final/sstcore-15.1.0.tar.gz
+SST_ELEMENTS_1510=https://github.com/sstsimulator/sst-elements/releases/download/v15.1.0_Final/sstelements-15.1.0.tar.gz
+TOOLDIR=${TOOLDIR:=@TOOLDIR@}
+
+DEBIAN_FRONTEND=noninteractive sudo apt install openmpi-bin openmpi-common libtool libtool-bin autoconf python3 python3-dev automake build-essential git
+wget $OPENMPI_416
+tar -xvzf openmpi-4.1.6.tar.gz
+mkdir -p $TOOLDIR && rm -rf $TOOLDIR/openmpi-4.1.6 && mv openmpi-4.1.6 $TOOLDIR
+rm -rf openmpi-4.1.6.tar.gz
+
+mkdir -p $TOOLDIR/openmpi_install
+cd $TOOLDIR/openmpi-4.1.6
+
+export MPIHOME=$TOOLDIR/openmpi_install
+./configure --prefix=$MPIHOME
+make all install
+
+export PATH=$MPIHOME/bin:$PATH
+export MPICC=mpicc
+export MPICXX=mpicxx
+
+echo 'export PATH='"$MPIHOME"'/bin:$PATH' >> ~/.bashrc
+echo 'export MPICC=mpicc' >> ~/.bashrc
+echo 'export MPICXX=mpicxx' >> ~/.bashrc
+
+rm -r $TOOLDIR/openmpi-4.1.6
+
+cd $TOOLDIR
+wget $SST_CORE_1510
+tar -xvzf sstcore-15.1.0.tar.gz
+rm sstcore-15.1.0.tar.gz
+cd sst-core
+
+mkdir -p $TOOLDIR/sst-install/sst-core
+export SST_CORE_HOME=$TOOLDIR/sst-install/sst-core
+export SST_CORE_ROOT=$TOOLDIR/sst-core
+echo 'export SST_CORE_HOME='"$SST_CORE_HOME" >> ~/.bashrc
+#echo 'export SST_CORE_ROOT='"$SST_CORE_ROOT" >> ~/.bashrc
+
+autoreconf -fi
+./configure --prefix=$SST_CORE_HOME
+make -j$(nproc) all
+make install
+
+rm -r $SST_CORE_ROOT
+
+export PATH=$SST_CORE_HOME/bin:$PATH
+echo 'export PATH='"$SST_CORE_HOME"'/bin:$PATH' >> ~/.bashrc
+
+cd $TOOLDIR
+wget $SST_ELEMENTS_1510
+tar -xvzf sstelements-15.1.0.tar.gz
+rm sstelements-15.1.0.tar.gz
+cd sst-elements
+
+mkdir -p $TOOLDIR/sst-install/sst-elements
+export SST_ELEMENTS_HOME=$TOOLDIR/sst-install/sst-elements
+export SST_ELEMENTS_ROOT=$TOOLDIR/sst-elements
+echo 'export SST_ELEMENTS_HOME='"$SST_ELEMENTS_HOME" >> ~/.bashrc
+#echo 'export SST_ELEMENTS_ROOT='"$SST_ELEMENTS_ROOT" >> ~/.bashrc
+
+./configure --prefix=$SST_ELEMENTS_HOME --with-sst-core=$SST_CORE_HOME
+make -j2 all
+make install
+
+rm -r $SST_ELEMENTS_ROOT
+
+export PATH=$SST_ELEMENTS_HOME/bin:$PATH
+echo 'export PATH='"$SST_ELEMENTS_HOME"'/bin:$PATH' >> ~/.bashrc
+
+if [ -n "$GITHUB_PATH" ]; then
+  echo "$SST_CORE_HOME/bin" >> "$GITHUB_PATH"
+  echo "$SST_ELEMENTS_HOME/bin" >> "$GITHUB_PATH"
+fi
+
+
+
+

ci/sst_test_vortex_conform.py

@@ -0,0 +1,7 @@
+import sst
+
+gpu = sst.Component("gpu0", "vortex.VortexGPGPU")
+gpu.addParams({
+    "clock": "1GHz",
+    "program": "tests/kernel/conform/conform.bin"
+})

ci/sst_test_vortex_fibonacci.py

@@ -0,0 +1,7 @@
+import sst
+
+gpu = sst.Component("gpu0", "vortex.VortexGPGPU")
+gpu.addParams({
+    "clock": "1GHz",
+    "program": "tests/kernel/fibonacci/fibonacci.bin"
+})

ci/sst_test_vortex_hello.py

@@ -0,0 +1,7 @@
+import sst
+
+gpu = sst.Component("gpu0", "vortex.VortexGPGPU")
+gpu.addParams({
+    "clock": "1GHz",
+    "program": "tests/kernel/hello/hello.bin"
+})

ci/sst_test_vortex_vecadd.py

@@ -0,0 +1,7 @@
+import sst
+
+gpu = sst.Component("gpu0", "vortex.VortexGPGPU")
+gpu.addParams({
+    "clock": "1GHz",
+    "program": "tests/kernel/vecadd/vecadd.bin"
+})

sim/simx/Makefile

@@ -1,6 +1,8 @@
 include ../common.mk
 
 DESTDIR ?= $(CURDIR)
+USE_SST ?= 0
+#SST_PKG ?= SST-14.1 # default SST package name
 
 OBJ_DIR = $(DESTDIR)/obj
 CONFIG_FILE = $(DESTDIR)/simx_config.stamp
@@ -27,6 +29,14 @@ SRCS += $(SRC_DIR)/execute.cpp $(SRC_DIR)/func_unit.cpp
 SRCS += $(SRC_DIR)/cache_sim.cpp $(SRC_DIR)/mem_sim.cpp $(SRC_DIR)/local_mem.cpp $(SRC_DIR)/mem_coalescer.cpp
 SRCS += $(SRC_DIR)/dcrs.cpp $(SRC_DIR)/types.cpp
 
+ifeq ($(USE_SST), 1)
+VORTEX_SST_SRCS := $(SRC_DIR)/vortex_simulator.cpp
+VORTEX_SST_SRCS += $(SRC_DIR)/VortexGPGPU.cpp
+SST_CFLAGS := $(shell sst-config --ELEMENT_CXXFLAGS)
+SST_LFLAGS := $(shell sst-config --ELEMENT_LDFLAGS)
+CXXFLAGS += $(SST_CFLAGS) -DUSE_SST
+endif
+
 # Add V extension sources
 ifneq ($(findstring -DEXT_V_ENABLE, $(CONFIGS)),)
   	SRCS += $(SRC_DIR)/voperands.cpp
@@ -65,14 +75,25 @@ DEPS := $(OBJS:.o=.d) $(MAIN_OBJ:.o=.d)
 # generate .d files alongside .o files
 CXXFLAGS += -MMD -MP -MF $(@:.o=.d)
 
+ifeq ($(USE_SST), 1)
+VORTEX_SST_OBJS := $(patsubst $(SRC_DIR)/%.cpp,$(OBJ_DIR)/%.o,$(VORTEX_SST_SRCS))
+DEPS += $(VORTEX_SST_OBJS:.o=.d)
+endif
+
+
 # optional: pipe through ccache if you have it
 CXX := $(if $(shell which ccache),ccache $(CXX),$(CXX))
 
 PROJECT := simx
+VORTEX_LIB := libvortex.so
 
-.PHONY: all force clean clean-lib clean-exe clean-obj
+.PHONY: all force clean clean-lib clean-exe clean-obj libvortex clean-libvortex
 
+ifeq ($(USE_SST), 1)
+all: $(DESTDIR)/$(PROJECT) $(DESTDIR)/$(VORTEX_LIB)
+else
 all: $(DESTDIR)/$(PROJECT)
+endif
 
 # build common object files
 $(OBJ_DIR)/common/%.o: $(SW_COMMON_DIR)/%.cpp $(CONFIG_FILE)
@@ -84,6 +105,11 @@ $(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp $(CONFIG_FILE)
 	@mkdir -p $(@D)
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+# build SST-specific source object files
+$(VORTEX_SST_OBJS): $(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp $(CONFIG_FILE)
+	@mkdir -p $(@D)
+	$(CXX) $(CXXFLAGS) $(SST_CFLAGS) -c $< -o $@
+
 # build main object file
 $(MAIN_OBJ): $(SRC_DIR)/main.cpp $(CONFIG_FILE)
 	@mkdir -p $(@D)
@@ -97,6 +123,16 @@ $(DESTDIR)/$(PROJECT): $(OBJS) $(MAIN_OBJ)
 $(DESTDIR)/lib$(PROJECT).so: $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -shared $(LDFLAGS) -o $@
 
+# Vortex SST simulator component shared library
+libvortex: $(DESTDIR)/$(VORTEX_LIB)
+
+$(DESTDIR)/$(VORTEX_LIB): $(OBJS) $(VORTEX_SST_OBJS)
+	$(CXX) $(CXXFLAGS) $(SST_CFLAGS) \
+	-I./sim/simx \
+	$(OBJS) $(VORTEX_SST_SRCS) \
+	-shared -o $@ \
+	$(LDFLAGS) $(SST_LFLAGS)
+
 # updates the timestamp when flags changed.
 $(CONFIG_FILE): force
 	@mkdir -p $(@D)
@@ -113,10 +149,13 @@ $(CONFIG_FILE): force
 clean-lib:
 	rm -f $(DESTDIR)/lib$(PROJECT).so
 
+clean-libvortex:
+	rm -f $(DESTDIR)/libvortex.so
+
 clean-exe:
 	rm -f $(DESTDIR)/$(PROJECT)
 
 clean-obj:
 	rm -rf $(OBJ_DIR)
 
-clean: clean-lib clean-exe clean-obj
+clean: clean-lib clean-exe clean-obj clean-libvortex

sim/simx/VortexGPGPU.cpp

@@ -0,0 +1,54 @@
+#include <sst/core/sst_config.h>
+#include "VortexGPGPU.h"
+#include <cstdlib>
+#include <vector>
+#include <utility>
+#include <unordered_map>
+
+using namespace SST;
+using namespace SST::Vortex;
+
+VortexGPGPU::VortexGPGPU(ComponentId_t id, Params &params)
+    : Component(id),
+      sim_(std::make_unique<vortex::VortexSimulator>()) {
+
+    std::cout << "VortexGPGPU Component: Initializing Vortex GPGPU simulator\n";
+
+    // Parameter: clock frequency (default 1GHz)
+    std::string clockfreq = params.find<std::string>("clock", "1GHz");
+
+    // Parameter: program path
+    std::string kernel = params.find<std::string>("program", "");
+
+    // Register our clock handler with SST
+    registerClock(clockfreq,
+                  new SST::Clock::Handler<VortexGPGPU>(this, &VortexGPGPU::clockTick));
+
+    // Load the kernel image
+    if (!sim_->init(kernel)) {
+        SST::Output out;
+        out.fatal(CALL_INFO, -1, "VortexSimulator init failed\n");
+    }
+    else{
+        std::cout << "VortexGPGPU Component: loaded kernel: " << kernel << std::endl;
+    }
+
+    registerAsPrimaryComponent();
+    primaryComponentDoNotEndSim();
+}
+
+VortexGPGPU::~VortexGPGPU() = default;
+
+void VortexGPGPU::setup() {}
+void VortexGPGPU::finish() {}
+
+// Advance the GPU execution one cycle based on SST clock handler callback
+bool VortexGPGPU::clockTick(SST::Cycle_t cycle) {
+    bool running = sim_->cycle();
+    if (!running) {
+        primaryComponentOKToEndSim();
+        std::cout << "VortexGPGPU Component: simulation finished\n";
+        return true;
+    }
+    return false;
+}