Build and install shared library for consumers and backends (pytorch#18560)

### Summary

These are the changes that I needed so that both my Mesa backend
(tentatively named Torx), and my test suite can link to Executorch. This
should be also what Linux distributions such as Debian and Fedora would
need to package Executorch.

There are some fixes as well for recent toolchains, whichi will be
needed as well by distributions.

I also install the headers for runtime/backend/, which weren't installed
before, to make out-of-tree backends to build cleanly.

### Test plan

Build with:

```
$ CMAKE_ARGS="-DEXECUTORCH_BUILD_SHARED=ON \
       -DEXECUTORCH_BUILD_EXTENSION_LLM=OFF \
       -DEXECUTORCH_BUILD_EXTENSION_LLM_RUNNER=OFF" \
       pip install --break-system-packages --no-build-isolation .
```
       
Then run:

```
$ cmake --install pip-out/temp.linux-$(uname -m)-cpython-3*/cmake-out
```

And check that the headers and shared library (libexecutorch.so.1.3.0)
were installed.

cc @kimishpatel @YifanShenSZ @cymbalrush @metascroy

---------

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Gregory Comer <gjcomer@meta.com>

Author: 3 people

CMakeLists.txt

@@ -46,7 +46,6 @@
 #
 
 cmake_minimum_required(VERSION 3.24)
-project(executorch)
 
 set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -87,6 +86,10 @@ if("${ET_VERSION_MAJOR}" STREQUAL ""
   )
 endif()
 
+project(executorch
+        VERSION "${ET_VERSION_MAJOR}.${ET_VERSION_MINOR}.${ET_VERSION_PATCH}"
+)
+
 message(
   STATUS
     "ExecuTorch version: ${ET_VERSION_MAJOR}.${ET_VERSION_MINOR}.${ET_VERSION_PATCH}"
@@ -177,6 +180,10 @@ if(DEFINED EXECUTORCH_BAREMETAL_SKIP_INSTALL
   )
 endif()
 
+if(EXECUTORCH_BUILD_SHARED)
+  set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+endif()
+
 # Enable ccache if available
 find_program(CCACHE_PROGRAM ccache)
 if(CCACHE_PROGRAM)
@@ -1165,6 +1172,49 @@ if(EXECUTORCH_BUILD_KERNELS_LLM)
   list(APPEND _executorch_kernels custom_ops_aot_lib)
 endif()
 
+# Consolidated shared library: bundles executorch_core plus commonly used
+# extensions into a single libexecutorch.so.
+if(EXECUTORCH_BUILD_SHARED)
+  executorch_add_shared_library(executorch_shared)
+  set_target_properties(
+    executorch_shared
+    PROPERTIES OUTPUT_NAME executorch
+               ARCHIVE_OUTPUT_NAME executorch_shared
+               EXPORT_NAME executorch-shared
+  )
+  target_include_directories(
+    executorch_shared PUBLIC ${_common_include_directories}
+  )
+  target_compile_definitions(
+    executorch_shared PUBLIC C10_USING_CUSTOM_GENERATED_MACROS
+  )
+  # Link executorch without WHOLE_ARCHIVE because its INTERFACE link options
+  # (from executorch_target_link_options_shared_lib) already force
+  # whole-archive. Link executorch_core explicitly since executorch only has a
+  # PRIVATE dep on it (symbols wouldn't propagate otherwise).
+  target_link_libraries(
+    executorch_shared PRIVATE executorch
+                              $<LINK_LIBRARY:WHOLE_ARCHIVE,executorch_core>
+  )
+  foreach(_ext_target
+          extension_data_loader extension_flat_tensor extension_named_data_map
+          extension_module_static extension_tensor
+  )
+    if(TARGET ${_ext_target})
+      target_link_libraries(
+        executorch_shared PRIVATE $<LINK_LIBRARY:WHOLE_ARCHIVE,${_ext_target}>
+      )
+    endif()
+  endforeach()
+  configure_file(
+    tools/cmake/executorch.pc.in ${CMAKE_CURRENT_BINARY_DIR}/executorch.pc
+    @ONLY
+  )
+  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/executorch.pc
+          DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
+  )
+endif()
+
 if(EXECUTORCH_BUILD_KERNELS_QUANTIZED)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/kernels/quantized)
   executorch_target_link_options_shared_lib(quantized_ops_lib)

kernels/portable/CMakeLists.txt

@@ -106,6 +106,12 @@ install(
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/kernels/portable/
 )
 
+if(EXECUTORCH_BUILD_SHARED)
+  executorch_add_shared_library(
+    executorch_portable_ops portable_ops_lib portable_kernels executorch_shared
+  )
+endif()
+
 # Build the portable custom ops AOT library for registering custom ops into
 # PyTorch. Requires find_package(Torch), which must be called at root scope
 # before this subdirectory is processed. Not targeting ARM_BAREMETAL as aot_lib

kernels/quantized/CMakeLists.txt

@@ -157,3 +157,10 @@ install(
   PUBLIC_HEADER
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/kernels/quantized/
 )
+
+if(EXECUTORCH_BUILD_SHARED)
+  executorch_add_shared_library(
+    executorch_quantized_ops quantized_ops_lib quantized_kernels
+    executorch_shared
+  )
+endif()

tools/cmake/Utils.cmake

@@ -211,3 +211,33 @@ function(executorch_target_copy_mlx_metallib target)
     endif()
   endif()
 endfunction()
+
+# Create and install a shared library composed from dependency libraries. The
+# target links the provided dependencies and carries VERSION/SOVERSION.
+function(executorch_add_shared_library target_name)
+  set(empty_source_name "${target_name}_empty.cpp")
+  file(
+    GENERATE
+    OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${empty_source_name}"
+    CONTENT "// intentionally empty\n"
+  )
+  add_library(
+    ${target_name} SHARED "${CMAKE_CURRENT_BINARY_DIR}/${empty_source_name}"
+  )
+  if(ARGN)
+    target_link_libraries(${target_name} PRIVATE ${ARGN})
+  endif()
+  set_target_properties(
+    ${target_name}
+    PROPERTIES VERSION "${PROJECT_VERSION}"
+               SOVERSION "${PROJECT_VERSION_MAJOR}"
+               LINKER_LANGUAGE CXX
+  )
+  install(
+    TARGETS ${target_name}
+    EXPORT ExecuTorchTargets
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+endfunction()

tools/cmake/executorch.pc.in

@@ -0,0 +1,10 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
+
+Name: ExecuTorch
+Description: On-device AI framework for PyTorch models
+Version: @PROJECT_VERSION@
+Cflags: -I${includedir} -I${includedir}/executorch/runtime/core/portable_type -I${includedir}/executorch/runtime/core/portable_type/c10 -DC10_USING_CUSTOM_GENERATED_MACROS
+Libs: -L${libdir} -lexecutorch

tools/cmake/preset/default.cmake

@@ -221,6 +221,10 @@ define_overridable_option(
   EXECUTORCH_BUILD_CPUINFO "Build cpuinfo library." BOOL
   ${_default_executorch_build_cpuinfo}
 )
+define_overridable_option(
+  EXECUTORCH_BUILD_SHARED "Build a consolidated ExecuTorch shared library" BOOL
+  OFF
+)
 
 # Threadpool size options. At most one can be specified. Note that the default
 # is managed in threadpool.cpp to allow the user to specify an alternate mode