Add -bazel and -bazel-dev options to DependencyInstaller and Build scripts#9762
Conversation
|
clang-tidy review says "All clean, LGTM! 👍" |
There was a problem hiding this comment.
Code Review
This pull request introduces Bazel support for building and dependency installation. The changes to Build.sh and DependencyInstaller.sh add new flags (-bazel, -bazel-dev) to use Bazel. My review focuses on improving the portability and robustness of these new scripts. I've suggested adding support for macOS in the dependency installer and improving shell script practices by removing an unnecessary eval. I also noted a point for future improvement regarding package manager support for GUI dependencies.
| _install_bazel() { | ||
| local bazel_prefix=${PREFIX:-"/usr/local"} | ||
| log "Checking Bazel (via bazelisk)" | ||
| if _command_exists "bazelisk"; then | ||
| log "bazelisk already installed, skipping." | ||
| INSTALL_SUMMARY+=("Bazel: system=found, required=any, status=skipped") | ||
| return | ||
| fi | ||
| local arch | ||
| arch=$(uname -m) | ||
| local bazelisk_arch="amd64" | ||
| if [[ "${arch}" == "aarch64" ]]; then | ||
| bazelisk_arch="arm64" | ||
| fi | ||
| ( | ||
| cd "${BASE_DIR}" | ||
| _execute "Downloading bazelisk..." curl -Lo bazelisk \ | ||
| "https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${bazelisk_arch}" | ||
| chmod +x bazelisk | ||
| _execute "Installing bazelisk..." mv bazelisk "${bazel_prefix}/bin/bazelisk" | ||
| ) | ||
| # Install xcb libraries needed for GUI support with Bazel builds | ||
| if _command_exists "apt-get"; then | ||
| _execute "Installing xcb libraries for GUI support..." \ | ||
| apt-get -y install --no-install-recommends \ | ||
| libxcb1-dev libxcb-util-dev libxcb-icccm4-dev libxcb-image0-dev \ | ||
| libxcb-keysyms1-dev libxcb-randr0-dev libxcb-render-util0-dev \ | ||
| libxcb-xinerama0-dev libxcb-xkb-dev | ||
| fi | ||
| INSTALL_SUMMARY+=("Bazel: system=none, required=latest, status=installed") | ||
| } |
There was a problem hiding this comment.
This function currently only supports installing bazelisk on Linux. On macOS, it will fail because it tries to download a Linux binary. Please add support for macOS, for example by using Homebrew (brew install bazelisk).
_install_bazel() {
local bazel_prefix=${PREFIX:-/usr/local}
log "Checking Bazel (via bazelisk)"
if _command_exists "bazelisk"; then
log "bazelisk already installed, skipping."
INSTALL_SUMMARY+=("Bazel: system=found, required=any, status=skipped")
return
fi
if [[ "$(uname -s)" == "Darwin" ]]; then
if ! _command_exists "brew"; then
error "Homebrew not found, which is required to install bazelisk on macOS."
fi
_execute "Installing bazelisk..." brew install bazelisk
else
local arch
arch=$(uname -m)
local bazelisk_arch="amd64"
if [[ "${arch}" == "aarch64" ]]; then
bazelisk_arch="arm64"
fi
(
cd "${BASE_DIR}"
_execute "Downloading bazelisk..." curl -Lo bazelisk \
"https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${bazelisk_arch}"
chmod +x bazelisk
_execute "Installing bazelisk..." mv bazelisk "${bazel_prefix}/bin/bazelisk"
)
# Install xcb libraries needed for GUI support with Bazel builds
if _command_exists "apt-get"; then
_execute "Installing xcb libraries for GUI support..." \
apt-get -y install --no-install-recommends \
libxcb1-dev libxcb-util-dev libxcb-icccm4-dev libxcb-image0-dev \
libxcb-keysyms1-dev libxcb-randr0-dev libxcb-render-util0-dev \
libxcb-xinerama0-dev libxcb-xkb-dev
fi
fi
INSTALL_SUMMARY+=("Bazel: system=none, required=latest, status=installed")
}| _install_bazel_dev() { | ||
| local bazel_prefix=${PREFIX:-"/usr/local"} | ||
| log "Checking Bazel dev tools (buildifier)" | ||
| if _command_exists "buildifier"; then | ||
| log "buildifier already installed, skipping." | ||
| INSTALL_SUMMARY+=("buildifier: system=found, required=any, status=skipped") | ||
| return | ||
| fi | ||
| local arch | ||
| arch=$(uname -m) | ||
| local buildifier_arch="amd64" | ||
| if [[ "${arch}" == "aarch64" ]]; then | ||
| buildifier_arch="arm64" | ||
| fi | ||
| ( | ||
| cd "${BASE_DIR}" | ||
| _execute "Downloading buildifier..." curl -Lo buildifier \ | ||
| "https://github.com/bazelbuild/buildtools/releases/latest/download/buildifier-linux-${buildifier_arch}" | ||
| chmod +x buildifier | ||
| _execute "Installing buildifier..." mv buildifier "${bazel_prefix}/bin/buildifier" | ||
| ) | ||
| INSTALL_SUMMARY+=("buildifier: system=none, required=latest, status=installed") | ||
| } |
There was a problem hiding this comment.
Similar to _install_bazel, this function only supports Linux. Please add support for macOS. buildifier can be installed with Homebrew (brew install buildifier).
_install_bazel_dev() {
local bazel_prefix=${PREFIX:-/usr/local}
log "Checking Bazel dev tools (buildifier)"
if _command_exists "buildifier"; then
log "buildifier already installed, skipping."
INSTALL_SUMMARY+=("buildifier: system=found, required=any, status=skipped")
return
fi
if [[ "$(uname -s)" == "Darwin" ]]; then
if ! _command_exists "brew"; then
error "Homebrew not found, which is required to install buildifier on macOS."
fi
_execute "Installing buildifier..." brew install buildifier
else
local arch
arch=$(uname -m)
local buildifier_arch="amd64"
if [[ "${arch}" == "aarch64" ]]; then
buildifier_arch="arm64"
fi
(
cd "${BASE_DIR}"
_execute "Downloading buildifier..." curl -Lo buildifier \
"https://github.com/bazelbuild/buildtools/releases/latest/download/buildifier-linux-${buildifier_arch}"
chmod +x buildifier
_execute "Installing buildifier..." mv buildifier "${bazel_prefix}/bin/buildifier"
)
fi
INSTALL_SUMMARY+=("buildifier: system=none, required=latest, status=installed")
}
etc/Build.sh
Outdated
| if command -v bazelisk &> /dev/null; then | ||
| bazel_cmd="bazelisk" | ||
| fi | ||
| eval "${bazel_cmd}" build //:openroad |
etc/DependencyInstaller.sh
Outdated
| if _command_exists "apt-get"; then | ||
| _execute "Installing xcb libraries for GUI support..." \ | ||
| apt-get -y install --no-install-recommends \ | ||
| libxcb1-dev libxcb-util-dev libxcb-icccm4-dev libxcb-image0-dev \ | ||
| libxcb-keysyms1-dev libxcb-randr0-dev libxcb-render-util0-dev \ | ||
| libxcb-xinerama0-dev libxcb-xkb-dev | ||
| fi |
…Build.sh Signed-off-by: alokkumardalei-wq <alokkumardalei2@gmail.com>
9a72b14 to
c2d59d2
Compare
|
clang-tidy review says "All clean, LGTM! 👍" |
1 similar comment
|
clang-tidy review says "All clean, LGTM! 👍" |
7739166 to
c2d59d2
Compare
|
clang-tidy review says "All clean, LGTM! 👍" |
|
Hello @maliberty the https://jenkins.openroad.tools/job/OpenROAD-Public/job/PR-9762-merge/4/display/redirect is failing at build 4 as I checked but has passed in build 1, 2 and 3 . I have no idea why is it so . Could you please look into this when you have time. Thank you. |
etc/Build.sh
Outdated
| if command -v bazelisk &> /dev/null; then | ||
| bazel_cmd="bazelisk" | ||
| fi | ||
| "${bazel_cmd}" build //:openroad |
There was a problem hiding this comment.
this should honor the GUI flag and numThreads, and should build release or opt
|
I think this is a bad idea. We want to ultimately phase out DependencyInstaller and install only bazel and let it pull in dependencies. DependencyInstaller should be kept as is for the CMake use-case, IMO. |
|
@oharboe I think this will provide for a smoother transition and if bazel can handle it all then maybe it can be retired in the future. (although we still need to install bazel/bazelisk and buildifier, so something is needed to get folks onboarded quickly and easily). |
| cd "${BASE_DIR}" | ||
| _execute "Downloading bazelisk..." curl -Lo bazelisk \ | ||
| "https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${bazelisk_arch}" | ||
| chmod +x bazelisk |
There was a problem hiding this comment.
need to verify the checksum _verify_checksum
| cd "${BASE_DIR}" | ||
| _execute "Downloading buildifier..." curl -Lo buildifier \ | ||
| "https://github.com/bazelbuild/buildtools/releases/latest/download/buildifier-linux-${buildifier_arch}" | ||
| chmod +x buildifier |
There was a problem hiding this comment.
need to verify the checksum _verify_checksum
etc/DependencyInstaller.sh
Outdated
| error "You must use one of: -all, -base, -common, -bazel, or -bazel-dev." | ||
| fi | ||
|
|
||
| if [[ "${INSTALL_BAZEL}" == "yes" ]]; then |
There was a problem hiding this comment.
probably should include bazel if doing -bazel-dev
"|| ${INSTALL_BAZEL_DEV}" == "yes"
|
Being a user of bazel, what I would be more interested in is:
This would probably require framework inverting DependencyInstaller so that it calls one decomposed build per tool. Claude should do that automatically. |
|
as for dev dependencies and linters and such, I think a better path is to add e.g. lint targets and let those lint targets handle dependencies in a bazel idiomatic fashion #9734 |
|
in that same vein check docs with bazel #9733 |
|
I think that this PR rightly identifies some real pain-points that I have myself. |
|
@alokkumardalei-wq @gadfort Thougths? The two PRs below build on the findings in this PR and should addresses the concerns: |
|
@alokkumardalei-wq Did you see my comments? #9762 (comment) |
|
clang-tidy review says "All clean, LGTM! 👍" |
83b0442 to
d853468
Compare
|
Thanks for the feedback! I have updated the PR to address @gadfort 's review comments: I added checksum verification for the tool downloads, made -bazel-dev automatically include the base -bazel tools, and updated Build.sh to respect the -no-gui and -threads flags while defaulting to -c opt." |
|
clang-tidy review says "All clean, LGTM! 👍" |
|
Thanks @oharboe for your suggestions, I see the architectural point about keeping DependencyInstaller.sh purely for CMake and moving Bazel dependency management entirely into bazel run targets (as proposed in PR #9772). Since @gadfort suggested this PR could serve as a useful transition step, I am open to whatever the core maintainers decide: either treating this as a temporary bridge while the Bazel-native targets mature, or closing this in favor of PR #9772. Thank you! |
|
@alokkumardalei-wq Please also read The-OpenROAD-Project/OpenROAD-flow-scripts#4003 and let me know what your thoughts are this PR in light of that PR getting accepted. |
|
Thanks @oharboe for linking that! I just reviewed OpenROAD-flow-scripts#4003. Yes, adding bazelisk run //:install to ORFS directly addresses the original paint-point I raised in #9594. It's a much cleaner, Bazel-native approach than trying to overload DependencyInstaller.sh and Build.sh. Thank you! |
|
@alokkumardalei-wq Is #9772 needed? I thought we were statically linked. Please advice if it should be re-opened with some specific instructions on what exactly it should test for. |
|
@alokkumardalei-wq @oharboe please keep in mind that not all users are using ORFS and we should not expecting that of users and developers. |
Sounds good. Please comment in my PR where we're not being use-case agnostic. I've tried to be agnostic w.r.t. nix, cmake and bazel. |
gadfort
left a comment
There was a problem hiding this comment.
I think just correct the flag and it should be ready for a full review. Thanks
etc/Build.sh
Outdated
| fi | ||
| bazelArgs=("-c" "opt" "--jobs=${numThreads}") | ||
| if [[ "$noGui" == "yes" ]]; then | ||
| bazelArgs+=("--define=enable_gui=false") |
|
Thanks @oharboe ,yes, I checked the Bazel dependency graph . The qt-bazel setup safely downloads xcb-proto and hermetically compiles/statically links the xcb plugins from source inside the sandbox. Because of this, it doesn't depend on system-level libxcb-*-dev packages being installed via apt. Therefore, PR #9772 is not needed but want to hear from you I am if getting wrong somewhere. Thank you! |
|
@alokkumardalei-wq Thanks for checking! Yes, best I know, #9772 is not needed. We can re-open it or something like it if we find later that it is needed. Close this one? |
|
@alokkumardalei-wq If approperiate, please consider leaving a note in The-OpenROAD-Project/OpenROAD-flow-scripts#4003, it will speed up reading for maintainers as they won't have to trawl through this issue to figure out that you prefer that solution too. |
|
Close? |
No, there is an issue I raised to do this, so no need to close, this should be merged once ready. |
|
My concerns are addressed by The-OpenROAD-Project/OpenROAD-flow-scripts#4003 I have no opinion about or stake in the DependencyInstaller.sh use-case as I view it as deprecated, but still important for the users. Those that still use it know what the pain points are. |
1. Add checksum verification to bazelisk and buildifier downloads. 2. Make -bazel-dev auto-install -bazel. 3. Update Build.sh Bazel command to honor -no-gui and -threads, and default to -c opt. Signed-off-by: alokkumardalei-wq <alokkumardalei2@gmail.com>
d853468 to
5af28c1
Compare
|
clang-tidy review says "All clean, LGTM! 👍" |
|
Hello @gadfort thanks for the feedback I think I have addressed all your suggestions/feedback , please have a review when you have time and let know if anything left to work on. Thank you! |
|
Hello @gadfort and @maliberty I think all CI checks are passing now , also I have done the changes suggested by you, please have a look onto this when you have time and please let me know If still I missed something critical or anything. Thank you. |
What does this PR solves?
Fixes #9594
Adds Bazel setup options to DependencyInstaller.sh and Build.sh to make it easier for developers to get started with the Bazel build.
Changes
DependencyInstaller.sh:
Both flags work standalone or combined with -all/-base/-common
Build.sh:
Testing/Verification:
Expected output:
Expected output: