Deeploy

.github/workflows/regression-tests.yml

@@ -0,0 +1,49 @@
+name: PULP-SDK CI
+on: [push, pull_request]
+jobs:
+  Regression-Tests:
+    name: Regression Tests
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "The job was automatically triggered by a ${{ github.event_name }} event."
+      - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub!"
+      - run: echo "The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."      
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - run: echo "The ${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "The workflow is now ready to test your code on the runner."
+      - name: Download PULP toolchain
+        run: |
+          wget https://github.com/pulp-platform/pulp-riscv-gnu-toolchain/releases/download/v1.0.16/v1.0.16-pulp-riscv-gcc-ubuntu-18.tar.bz2
+          tar -xf v1.0.16-pulp-riscv-gcc-ubuntu-18.tar.bz2
+          cd v1.0.16-pulp-riscv-gcc-ubuntu-18
+          export PULP_RISCV_GCC_TOOLCHAIN=$(pwd)
+          cd ${{ github.workspace }}
+          export -p > temp_env
+      - name: Install PULP-SDK dependancies
+        run: |
+          source temp_env
+          cd ${{ github.workspace }}
+          sudo apt-get update -y
+          sudo apt-get install -y build-essential git libftdi-dev libftdi1 doxygen python3-pip libsdl2-dev curl cmake libusb-1.0-0-dev scons gtkwave libsndfile1-dev rsync autoconf automake texinfo libtool pkg-config libsdl2-ttf-dev
+          pip install --user argcomplete pyelftools
+          cd ${{ github.workspace }}
+          export -p > temp_env
+      - name: Build PULP-SDK
+        run: |
+          source temp_env
+          source configs/pulp-open.sh
+          make build
+          cd ${{ github.workspace }}
+          export -p > temp_env
+      - name: Launch regression tests
+        run: |
+          source temp_env
+          cd tests/
+          dirs=$(dirname $(find . -name "Makefile" -printf "%P\n")); for file in ${dirs}; do cd ${file}; echo ${file}; rm -rf BUILD/; make clean all run; cd -; done;
+          cd ${{ github.workspace }}
+          export -p > temp_env
+      - name: List files in the repository
+        run: |
+          ls ${{ github.workspace }}
+      - run: echo "This job's status is ${{ job.status }}."

.gitignore

@@ -0,0 +1,5 @@
+__pycache__
+BUILD/
+build/
+install/
+.sconsign.dblite

Makefile

@@ -1,14 +1,18 @@
+# the MAGICK_ROOT variable can be used to pass an alternative installation
+# prefix for the GraphicsMagick library. 
+ifdef MAGICK_ROOT
+export MAGICK_ROOT := $(realpath $(MAGICK_ROOT))
+endif
+
 SHELL=bash
 
-ifndef GAP_SDK_HOME
+ifndef PULP_SDK_HOME
     $(error Please source the proper configuration first)
 endif
 
 BUILD_DIR ?= $(CURDIR)/build
-ARTIFACT_PATH ?= $(CURDIR)/artifact
 
 export BUILD_DIR
-export ARTIFACT_PATH
 
 include rules/json-tools.mk
 include rules/gap-configs.mk
@@ -17,19 +21,8 @@ include rules/dpi-models.mk
 include rules/gvsoc.mk
 include rules/pulpos.mk
 
-checkout: gvsoc.checkout.all
-	git submodule update --init rtos/pulpos/pulp_hal rtos/pulpos/pulp_archi tests/pmsis_tests rtos/pulpos/pulp rtos/pmsis/pmsis_api tools/gapy
-
 build: gvsoc.build.all
 
 clean: gvsoc.clean
 
-all: checkout build
-
-artifact: gvsoc.artifact pulpos.artifact
-	rsync -avR --exclude=".git*" Makefile rules configs tools/rules README.md LICENSE $(ARTIFACT_PATH)
-
-	mkdir -p $(ARTIFACT_PATH)/tools/jenkins
-	git rev-parse HEAD &> $(ARTIFACT_PATH)/tools/jenkins/sdk_version.txt
-
-.PHONY: artifact
+all: build

README.md

@@ -1,31 +1,86 @@
-# SDK setup
-
-Previous version is available at: https://github.com/pulp-platform/pulp-sdk-legacy
+# PULP-SDK
+
+This is the latest version of the PULP SDK, which is under active development. The previous (now legacy) version, which is no longer supported, is on the [`v1` branch](https://github.com/pulp-platform/pulp-sdk/tree/v1).
+
+## Citing
+
+If you intend to use or reference GVSoC for an academic publication, please consider citing it:
+
+```
+@INPROCEEDINGS{9643828,
+	author={Bruschi, Nazareno and Haugou, Germain and Tagliavini, Giuseppe and Conti, Francesco and Benini, Luca and Rossi, Davide},
+	booktitle={2021 IEEE 39th International Conference on Computer Design (ICCD)},
+	title={GVSoC: A Highly Configurable, Fast and Accurate Full-Platform Simulator for RISC-V based IoT Processors},
+	year={2021},
+	volume={},
+	number={},
+	pages={409-416},
+	doi={10.1109/ICCD53106.2021.00071}}
+```
+
+PULP-SDK completely supports Dory as QNNs deployment tool. You can find an example of deployment [\[here\]](#cnns-at-the-edge). If you intend to use or reference Dory for an academic publication, please consider citing it:
+
+```
+@ARTICLE{9381618,
+	author={Burrello, Alessio and Garofalo, Angelo and Bruschi, Nazareno and Tagliavini, Giuseppe and Rossi, Davide and Conti, Francesco},
+	journal={IEEE Transactions on Computers},
+	title={DORY: Automatic End-to-End Deployment of Real-World DNNs on Low-Cost IoT MCUs},
+	year={2021},
+	volume={70},
+	number={8},
+	pages={1253-1268},
+	doi={10.1109/TC.2021.3066883}}
+```
 
 ## Getting started
 
-These instructions were developed using a fresh Ubuntu 18.04 Bionic Beaver 64-Bit virtual machine from https://www.osboxes.org/ubuntu/#ubuntu-1804-info
+These instructions were developed using a fresh Ubuntu 18.04 Bionic Beaver 64-Bit.
 
 The following packages needed to be installed:
 
 ~~~~~shell
 sudo apt-get install -y build-essential git libftdi-dev libftdi1 doxygen python3-pip libsdl2-dev curl cmake libusb-1.0-0-dev scons gtkwave libsndfile1-dev rsync autoconf automake texinfo libtool pkg-config libsdl2-ttf-dev
 ~~~~~
 
+The SDK also requires the `argcomplete` and `pyelftools` Python package. You can install them for the local user with:
+~~~~~shell
+pip install --user argcomplete pyelftools
+~~~~~
+Omit `--user` to install at system level instead, which will probably require admin rights.
+
+This version requires PULP toolchain to compile the application exploiting pulp features. PULP toolchain is available at: https://github.com/pulp-platform/pulp-riscv-gnu-toolchain
+
+You can choose also its precompiled version, exploring: https://github.com/pulp-platform/pulp-riscv-gnu-toolchain/releases/tag/v1.0.16
+
+Please, refer to the corresponding README for the installation.
+
+Once PULP toolchain is correctly installed, define the path in which there is toolchain bin folder:
+
+~~~~~shell
+export PULP_RISCV_GCC_TOOLCHAIN=<INSTALL_DIR>
+~~~~~
+
 Source the file corresponding to the desired configuration:
 
 ~~~~~shell
+cd pulp-sdk
 source configs/pulp-open.sh
 ~~~~~
 
 At least gcc 4.9.1 is needed. If the default one is not correct, CC and CXX can be set to
-point to a correct one.
+point to a correct one. To check if gcc has the right version:
+
+~~~~~shell
+gcc --version
+~~~~~
 
-GAP_RISCV_GCC_TOOLCHAIN must be defined to point to the riscv compiler.
+Please, refer to official guide to update gcc if is needed.
 
-## SDK build
+## GVSoC
 
-Compile the SDK with this command:
+GVSoC is the PULP chips simulator that is natively included in the SDK and is described and evaluated fully in Bruschi et al. [\[arXiv:2201.08166v1\]](https://arxiv.org/abs/2201.08166).
+
+Compile GVSoC with this command:
 
 ~~~~~shell
 make build
@@ -41,24 +96,60 @@ Then, go to a test, for example pmsis_tests/quick/cluster/fork/, and execute:
 make clean all run
 ~~~~~
 
-This will by default execute it on gvsoc, and you can configure the platform with this command:
+This will by default execute it on GVSoC (platform=gvsoc), and you can configure the RTL platform with this command:
 
 ~~~~~shell
 make clean all run platform=rtl
 ~~~~~
 
-## Getting a single module
+Notice that the environment variable `VSIM_PATH` should be set to the directory where the RTL platform has been built.
+This is typically done by sourcing the `setup/vsim.sh` file from the main folder of the RTL platform.
+
+<a name='cnns-at-the-edge'>
+## Application: CNNs at the Edge
+</a>
 
-You can also use this SDK to just get one module and his depencies, by doing:
+To run pre-generated real-world networks, such as MobileNetV1:
 
 ~~~~~shell
-make <module name>.all
+cd applications/MobileNetV1
+make clean all run platform=<PLATFORM> CORE=<NUM_CORES>
 ~~~~~
 
-For example, to get GVSOC, you can execute:
+### Nemo + Dory + Pulp-NN
+
+Our vertical flow allows to deploy optimized QNNs on low-power and low-resources MCUs, starting from a Pytorch model.
+
+#### Nemo
+
+[\[Nemo\]](https://github.com/pulp-platform/nemo) is a framework for Deep Neural Networks layer-wise quantization.
+He starts from a common Pytorch project and produces an equivalent quantized model, which well suits the usually integer MCUs.
+Its output are a `.onnx` as quantized model and several `.txt` as set of input and weigths of the network, also including the golden activations to checks the output of every network's layer.
+Please refer to its README for more details and [\[here\]](https://colab.research.google.com/drive/1qdc__9uZAGk9TzylsH3S8tB73bWcA2cA?usp=sharing#scrollTo=xelcF1jxkAit) you can find a Colab project and a very detailed tutorial on how to get started with Nemo.
+
+#### Dory
+
+[\[Dory\]](https://github.com/pulp-platform/dory) is an automatic tool to generate and directly deploy MLP/CNNs on PULP family boards, exploiting [\[Pulp-NN\]](https://github.com/pulp-platform/pulp-nn) as optimized back-end.
+
+Dory has a complete and autonomous testsuite, named [\[Dory-Example\]](https://github.com/pulp-platform/dory_examples), which is periodically updated, and please refer to its README for more details.
+To generate the code and run one of these examples:
+
+~~~~~shell
+cd dory/dory_examples/
+python3 network_generate --network_dir <e.g., ./examples/MobileNetV1/>
+cd application
+make clean all run platform=<PLATFORM> CORE=<NUM_CORES>
+~~~~~
+
+where you should choose `CORE=8` if you want to test the network on pulp cluster with all of the eight cores active (by default only 1 is set).
+
+To set up and execute a custom application, firstly, copy your file `network.onnx` and files `out_layer{i}.txt` in a single folder (e.g., `pulp-sdk/application/MyCustomNetwork/`) and then:
 
 ~~~~~shell
-make gvsoc.all
+cd dory/dory_examples/
+python3 network_generate --network_dir <pulp-sdk/application/MyCustomNetwork/>
+cd application
+make clean all run platform=<PLATFORM> CORE=<NUM_CORES>
 ~~~~~
 
-Note that you still need to source a config before doing that.
+You can use L1 and L2 memory constraints to specify the amount of memory used inside the application. Please refer to Dory and Dory-example READMEs for more details.

applications/MobileNetV1/DORY_network/inc/dory.h

@@ -0,0 +1,97 @@
+/*
+ * dory.h
+ * Alessio Burrello <alessio.burrello@unibo.it>
+ *
+ * Copyright (C) 2019-2020 University of Bologna
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. 
+ */
+
+#include "mchan_test.h"
+unsigned int dory_get_tile_1d(
+  unsigned x,
+  int tile_ii,
+  int tile_size_i,
+  int data_size
+);
+unsigned int dory_get_tile_2d(
+  unsigned int x,
+  int tile_ii,
+  int tile_jj,
+  int tile_size_i,
+  int tile_size_j,
+  int tile_stride_j,
+  int data_size
+);
+unsigned int dory_get_tile_3d(
+  unsigned int x,
+  int tile_ii,
+  int tile_jj,
+  int tile_kk,
+  int tile_size_i,
+  int tile_size_j,
+  int tile_size_k,
+  int tile_stride_j,
+  int tile_stride_k,
+  int tile_overlap_i,
+  int tile_overlap_j,
+  int tile_overlap_k,
+  int tile_offset_i,
+  int tile_offset_j,
+  int tile_offset_k,
+  int data_size
+);
+
+unsigned int dory_get_tile_4d(
+  unsigned int x,
+  int tile_ii,
+  int tile_jj,
+  int tile_kk,
+  int tile_ll,
+  int tile_size_i,
+  int tile_size_j,
+  int tile_size_k,
+  int tile_size_l,
+  int tile_stride_j,
+  int tile_stride_k,
+  int tile_stride_l,
+  int tile_offset_i,
+  int tile_offset_j,
+  int tile_offset_k,
+  int tile_offset_l,
+  int data_size
+);
+
+void dory_dma_memcpy_3d_custom(
+  unsigned int ext,
+  unsigned int loc,
+  unsigned short size,
+  unsigned short stride_1,
+  unsigned short stride_0,
+  unsigned short length_2,
+  unsigned short length_0,
+  unsigned int dir,
+  unsigned int *id
+);
+
+void dory_dma_memcpy_3d_custom_hwc_to_chw(
+  unsigned int ext,
+  unsigned int loc,
+  unsigned short size,
+  unsigned short stride_1,
+  unsigned short stride_0,
+  unsigned short length_2,
+  unsigned short length_0,
+  unsigned int dir,
+  unsigned int *id
+);