Fails to build on Apple M1

[certik]

I used the following on Apple M1 MacBook Pro:

$ conda create -n seq llvmdev=12 cmake
$ conda activate seq
$ cmake .. -DCMAKE_BUILD_TYPE=Release \         
                      -DLLVM_DIR=$(llvm-config --cmakedir) \
                      -DCMAKE_C_COMPILER=clang \
                      -DCMAKE_CXX_COMPILER=clang++
$ make
...
[  0%] Building C object _deps/openmp-build/runtime/src/CMakeFiles/omp.dir/z_Linux_asm.S.o
/Users/certik/repos/seq/build/_deps/openmp-src/runtime/src/z_Linux_asm.S:1546:5: error: unknown directive
    .size __kmp_unnamed_critical_addr,8
    ^

It's trying to compile x86 Linux assembly on macOS arm.

[certik]

This error seems the same as in spack/spack#26319. In there it is recommended to use LLVM 12.0.1, which is exactly the version that Conda installed above.

[certik]

The following patch makes it compile:

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ff29ed08..a93a89c2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -44,18 +44,12 @@ set(SEQRT_FILES
     runtime/lib.h
     runtime/lib.cpp
     runtime/exc.cpp
-    runtime/sw/ksw2.h
-    runtime/sw/ksw2_extd2_sse.cpp
-    runtime/sw/ksw2_exts2_sse.cpp
-    runtime/sw/ksw2_extz2_sse.cpp
-    runtime/sw/ksw2_gg2_sse.cpp
-    runtime/sw/intersw.h
-    runtime/sw/intersw.cpp)
+    )
 add_library(seqrt SHARED ${SEQRT_FILES})
 add_dependencies(seqrt bz2 liblzma zlibstatic gc htslib backtrace)
 set_source_files_properties(runtime/sw/intersw.cpp PROPERTIES COMPILE_FLAGS -mavx)
 target_include_directories(seqrt PRIVATE ${backtrace_SOURCE_DIR} "${gc_SOURCE_DIR}/include" runtime)
-target_link_libraries(seqrt PRIVATE omp backtrace ${STATIC_LIBCPP} LLVMSupport)
+target_link_libraries(seqrt PRIVATE backtrace ${STATIC_LIBCPP} LLVMSupport)
 if(APPLE)
     target_link_libraries(seqrt PRIVATE
         -Wl,-force_load,$<TARGET_FILE:bz2>
@@ -73,9 +67,9 @@ else()
         $<TARGET_FILE:htslib>
         -Wl,--no-whole-archive)
 endif()
-add_custom_command(TARGET seqrt POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:omp> ${CMAKE_BINARY_DIR})
-
+#add_custom_command(TARGET seqrt POST_BUILD
+#    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:omp> ${CMAKE_BINARY_DIR})
+#
 # Seq compiler library
 include_directories(${LLVM_INCLUDE_DIRS})
 add_definitions(${LLVM_DEFINITIONS})
diff --git a/runtime/lib.cpp b/runtime/lib.cpp
index e642e3ef..0a4e8ae6 100644
--- a/runtime/lib.cpp
+++ b/runtime/lib.cpp
@@ -17,7 +17,7 @@
 
 #define GC_THREADS
 #include "lib.h"
-#include "sw/ksw2.h"
+//#include "sw/ksw2.h"
 #include <gc.h>
 
 using namespace std;
@@ -53,7 +53,7 @@ SEQ_FUNC void seq_init(int d) {
   GC_set_warn_proc(GC_ignore_warn_proc);
   GC_allow_register_threads();
   // equivalent to: #pragma omp parallel { register_thread }
-  __kmpc_fork_call(&dummy_loc, 0, (kmpc_micro)register_thread);
+  //__kmpc_fork_call(&dummy_loc, 0, (kmpc_micro)register_thread);
   seq_exc_init();
   debug = d;
 }
@@ -342,12 +342,14 @@ struct Alignment {
 SEQ_FUNC void seq_align(seq_t query, seq_t target, int8_t *mat, int8_t gapo,
                         int8_t gape, seq_int_t bandwidth, seq_int_t zdrop,
                         seq_int_t end_bonus, seq_int_t flags, Alignment *out) {
-  ksw_extz_t ez;
+//  ksw_extz_t ez;
   ALIGN_ENCODE(encode);
+  /*
   ksw_extz2_sse(nullptr, qlen, qbuf, tlen, tbuf, 5, mat, gapo, gape, (int)bandwidth,
                 (int)zdrop, end_bonus, (int)flags, &ez);
+                */
   ALIGN_RELEASE();
-  *out = {{ez.cigar, ez.n_cigar}, flags & KSW_EZ_EXTZ_ONLY ? ez.max : ez.score};
+//  *out = {{ez.cigar, ez.n_cigar}, flags & KSW_EZ_EXTZ_ONLY ? ez.max : ez.score};
 }
 
 SEQ_FUNC void seq_align_default(seq_t query, seq_t target, Alignment *out) {
@@ -357,33 +359,22 @@ SEQ_FUNC void seq_align_default(seq_t query, seq_t target, Alignment *out) {
   int n_cigar = 0;
   uint32_t *cigar = nullptr;
   ALIGN_ENCODE(encode);
-  int score = ksw_gg2_sse(nullptr, qlen, qbuf, tlen, tbuf, 5, mat, 0, 1, -1, &m_cigar,
-                          &n_cigar, &cigar);
   ALIGN_RELEASE();
-  *out = {{cigar, n_cigar}, score};
 }
 
 SEQ_FUNC void seq_align_dual(seq_t query, seq_t target, int8_t *mat, int8_t gapo1,
                              int8_t gape1, int8_t gapo2, int8_t gape2,
                              seq_int_t bandwidth, seq_int_t zdrop, seq_int_t end_bonus,
                              seq_int_t flags, Alignment *out) {
-  ksw_extz_t ez;
   ALIGN_ENCODE(encode);
-  ksw_extd2_sse(nullptr, qlen, qbuf, tlen, tbuf, 5, mat, gapo1, gape1, gapo2, gape2,
-                (int)bandwidth, (int)zdrop, end_bonus, (int)flags, &ez);
   ALIGN_RELEASE();
-  *out = {{ez.cigar, ez.n_cigar}, flags & KSW_EZ_EXTZ_ONLY ? ez.max : ez.score};
 }
 
 SEQ_FUNC void seq_align_splice(seq_t query, seq_t target, int8_t *mat, int8_t gapo1,
                                int8_t gape1, int8_t gapo2, int8_t noncan,
                                seq_int_t zdrop, seq_int_t flags, Alignment *out) {
-  ksw_extz_t ez;
   ALIGN_ENCODE(encode);
-  ksw_exts2_sse(nullptr, qlen, qbuf, tlen, tbuf, 5, mat, gapo1, gape1, gapo2, noncan,
-                (int)zdrop, (int)flags, &ez);
   ALIGN_RELEASE();
-  *out = {{ez.cigar, ez.n_cigar}, flags & KSW_EZ_EXTZ_ONLY ? ez.max : ez.score};
 }
 
 SEQ_FUNC void seq_align_global(seq_t query, seq_t target, int8_t *mat, int8_t gapo,
@@ -393,21 +384,14 @@ SEQ_FUNC void seq_align_global(seq_t query, seq_t target, int8_t *mat, int8_t ga
   int n_cigar = 0;
   uint32_t *cigar = nullptr;
   ALIGN_ENCODE(encode);
-  int score = ksw_gg2_sse(nullptr, qlen, qbuf, tlen, tbuf, 5, mat, gapo, gape,
-                          (int)bandwidth, &m_cigar, &n_cigar, &cigar);
   ALIGN_RELEASE();
-  *out = {{backtrace ? cigar : nullptr, backtrace ? n_cigar : 0}, score};
 }
 
 SEQ_FUNC void seq_palign(seq_t query, seq_t target, int8_t *mat, int8_t gapo,
                          int8_t gape, seq_int_t bandwidth, seq_int_t zdrop,
                          seq_int_t end_bonus, seq_int_t flags, Alignment *out) {
-  ksw_extz_t ez;
   ALIGN_ENCODE(pencode);
-  ksw_extz2_sse(nullptr, qlen, qbuf, tlen, tbuf, 23, mat, gapo, gape, (int)bandwidth,
-                (int)zdrop, end_bonus, (int)flags, &ez);
   ALIGN_RELEASE();
-  *out = {{ez.cigar, ez.n_cigar}, flags & KSW_EZ_EXTZ_ONLY ? ez.max : ez.score};
 }
 
 SEQ_FUNC void seq_palign_default(seq_t query, seq_t target, Alignment *out) {
@@ -440,24 +424,16 @@ SEQ_FUNC void seq_palign_default(seq_t query, seq_t target, Alignment *out) {
       -1, -1, -1, -2, -3, -2, -3, -2, 3,  -3, 2,  -1, -2, -1, -1, -2, -3, -1, -2, -2,
       -2, -1, 2,  -1, 7,  -2, -1, 1,  -3, 1,  4,  -3, -2, 0,  -3, 1,  -3, -1, 0,  -1,
       3,  0,  0,  -1, -2, -3, -1, -2, 4};
-  ksw_extz_t ez;
   ALIGN_ENCODE(pencode);
-  ksw_extz2_sse(nullptr, qlen, qbuf, tlen, tbuf, 23, mat, 11, 1, -1, -1,
-                /* end_bonus */ 0, 0, &ez);
   ALIGN_RELEASE();
-  *out = {{ez.cigar, ez.n_cigar}, ez.score};
 }
 
 SEQ_FUNC void seq_palign_dual(seq_t query, seq_t target, int8_t *mat, int8_t gapo1,
                               int8_t gape1, int8_t gapo2, int8_t gape2,
                               seq_int_t bandwidth, seq_int_t zdrop, seq_int_t end_bonus,
                               seq_int_t flags, Alignment *out) {
-  ksw_extz_t ez;
   ALIGN_ENCODE(pencode);
-  ksw_extd2_sse(nullptr, qlen, qbuf, tlen, tbuf, 23, mat, gapo1, gape1, gapo2, gape2,
-                (int)bandwidth, (int)zdrop, end_bonus, (int)flags, &ez);
   ALIGN_RELEASE();
-  *out = {{ez.cigar, ez.n_cigar}, flags & KSW_EZ_EXTZ_ONLY ? ez.max : ez.score};
 }
 
 SEQ_FUNC void seq_palign_global(seq_t query, seq_t target, int8_t *mat, int8_t gapo,
@@ -466,10 +442,7 @@ SEQ_FUNC void seq_palign_global(seq_t query, seq_t target, int8_t *mat, int8_t g
   int n_cigar = 0;
   uint32_t *cigar = nullptr;
   ALIGN_ENCODE(pencode);
-  int score = ksw_gg2_sse(nullptr, qlen, qbuf, tlen, tbuf, 23, mat, gapo, gape,
-                          (int)bandwidth, &m_cigar, &n_cigar, &cigar);
   ALIGN_RELEASE();
-  *out = {{cigar, n_cigar}, score};
 }
 
 SEQ_FUNC bool seq_is_macos() {
diff --git a/scripts/deps.cmake b/scripts/deps.cmake
index b1585a10..ca7e131c 100644
--- a/scripts/deps.cmake
+++ b/scripts/deps.cmake
@@ -36,13 +36,6 @@ CPMAddPackage(
             "enable_thread_local_alloc ON"
             "enable_handle_fork ON")
 set_target_properties(cord PROPERTIES EXCLUDE_FROM_ALL ON)
-CPMAddPackage(
-    NAME openmp
-    GITHUB_REPOSITORY "llvm-mirror/openmp"
-    VERSION 9.0
-    GIT_TAG release_90
-    OPTIONS "OPENMP_ENABLE_LIBOMPTARGET OFF"
-            "OPENMP_STANDALONE_BUILD ON")
 CPMAddPackage(
     NAME backtrace
     GITHUB_REPOSITORY "ianlancetaylor/libbacktrace"

But when I try to compile a simple project, it fails with:

$ ./seqc build ../a.py    
ld: library not found for -lseqrt
clang-12: error: linker command failed with exit code 1 (use -v to see invocation)
error: process for 'clang' exited with status 1

[certik]

The last error can be mitigated by:

$ LD_LIBRARY_PATH=. ./seqc build ../a.py
ld: library not found for -lomp
clang-12: error: linker command failed with exit code 1 (use -v to see invocation)

[certik]

This seems to be a better patch:

--- a/compiler/sir/llvm/llvisitor.cpp
+++ b/compiler/sir/llvm/llvisitor.cpp
@@ -485,8 +485,7 @@ void LLVMVisitor::writeToExecutable(const std::string &filename,
   for (const auto &lib : libs) {
     command.push_back("-l" + lib);
   }
-  std::vector<std::string> extraArgs = {"-lseqrt", "-lomp", "-lpthread", "-ldl",
-                                        "-lz",     "-lm",   "-lc",       "-o",
+  std::vector<std::string> extraArgs = {"-L.", "-lseqrt", "-o",
                                         filename,  objFile};
   for (const auto &arg : extraArgs) {
     command.push_back(arg);

Now it fails with:

$ ./seqc build ../a.py 
ld: library not found for -lSystem
clang-12: error: linker command failed with exit code 1 (use -v to see invocation)
error: process for 'clang' exited with status 1

I am going to leave it at this. It seems it's pretty close, but not quite there yet to work on M1.

[markhend]

Hi @certik. Thanks for the efforts and information.

[arshajii]

Hi @certik — actually I’ll have access to an M1 Mac in a day or two so I can help debug this. Let me circle back with you then!

[arshajii]

I did have a chance to look into this a bit more. The original build error is from OpenMP, and is resolved in newer versions, so that in itself isn't a huge issue. However, it seems LLVM itself still has some catching up to do when it comes to M1, since exceptions aren't catchable in JIT mode yet (see also this issue), which causes a lot of problems. OpenMP also has some issues on M1, e.g. tasking causes a crash on shutdown; I observed a variant of this bug myself during testing as well.

In short, I was able to get a version of Seq up and running on M1, but it probably makes sense to wait for better support from LLVM first.