Skip to content

Unable to compile on RedHat 9 #1247

Open

Description

@juan131

I'm trying to compile Cilium Proxy on RedHat UBI 9 and aarch64 architecture. To do so, I patched https://github.com/cilium/proxy/blob/main/bazel/toolchains/BUILD as shown below to adapt include directories and link options:

diff --git a/bazel/toolchains/BUILD b/bazel/toolchains/BUILD
index f0cbb5c3..be6322e7 100644
--- a/bazel/toolchains/BUILD
+++ b/bazel/toolchains/BUILD
@@ -49,10 +49,13 @@ cc_toolchain_config(
     coverage_link_flags = ["--coverage"],
     cpu = "aarch64",
     cxx_builtin_include_directories = [
-        "/usr/lib/llvm-17",
-        "/usr/aarch64-linux-gnu/include",
-        "/usr/include",
-    ],
+        "/usr/lib/clang/18/include",
+        "/usr/include",
+        "/opt/rh/gcc-toolset-13/root/usr/local/include",
+        "/opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/include",
+        "/opt/rh/gcc-toolset-13/root/usr/include/c++/13",
+        "/opt/rh/gcc-toolset-13/root/usr/include/c++/13/aarch64-redhat-linux",
+    ],
     cxx_flags = ["-std=c++0x"],
     dbg_compile_flags = ["-g"],
     host_system_name = "local",
@@ -61,6 +64,8 @@ cc_toolchain_config(
         "-fuse-ld=lld",
         "-Wl,-no-as-needed",
         "-Wl,-z,relro,-z,now",
+        "-L/opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13",
+        "-Wl,-rpath,/opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13",
         "-lm",
     ],
     link_libs = ["-l:libstdc++.a"],
@@ -77,18 +82,18 @@ cc_toolchain_config(
     target_libc = "glibc",
     target_system_name = "aarch64-linux-gnu",
     tool_paths = {
-        "ar": "/usr/bin/llvm-ar-17",
-        "compat-ld": "/usr/bin/lld-17",
-        "ld": "/usr/bin/lld-17",
-        "gold": "/usr/bin/lld-17",
-        "cpp": "/usr/bin/clang-cpp-17",
-        "gcc": "/usr/bin/clang-17",
-        "dwp": "/usr/bin/llvm-dwp-17",
-        "gcov": "/usr/bin/llvmcov-17",
-        "nm": "/usr/bin/llvm-nm-17",
-        "objcopy": "/usr/bin/llvm-objcopy-17",
-        "objdump": "/usr/bin/llvm-objdump-17",
-        "strip": "/usr/bin/llvm-strip-17",
+        "ar": "/usr/bin/llvm-ar",
+        "compat-ld": "/usr/bin/lld",
+        "ld": "/usr/bin/lld",
+        "gold": "/usr/bin/lld",
+        "cpp": "/usr/bin/clang-cpp",
+        "gcc": "/usr/bin/clang",
+        "dwp": "/usr/bin/llvm-dwp",
+        "gcov": "/usr/bin/llvm-cov",
+        "nm": "/usr/bin/llvm-nm",
+        "objcopy": "/usr/bin/llvm-objcopy",
+        "objdump": "/usr/bin/llvm-objdump",
+        "strip": "/usr/bin/llvm-strip",
     },
     toolchain_identifier = "linux_aarch64",
     unfiltered_compile_flags = [

I have RedHat UBI 9 environment where I installed some system packages & bazellisk as shown below

dnf update -y
dnf --allowerasing install -y java-1.8.0-openjdk-headless clang-devel llvm-devel lld libstdc++-static

git clone git@github.com:cilium/proxy.git && cd proxy
./tools/install_bazelisk.sh

Commands I run:

./bazel/setup_clang.sh /usr
echo "build --config=libc++" > .bazelrc
bazel build \
      --config=release \
      --strip=always \
      --verbose_failures \
      --extra_toolchains=//bazel/toolchains:all \
      --noincompatible_strict_action_env \
      //:cilium-envoy

The error I'm facing:

ERROR: /home/cilium/.cache/bazel/_bazel_cilium/0211e60aab28805245749726eda1a88a/external/com_google_cel_cpp/common/BUILD:76:11: Compiling common/json.cc failed: (Exit 1): clang failed: error executing command (from target @com_google_cel_cpp//common:json) 
   (cd /home/cilium/.cache/bazel/_bazel_cilium/0211e60aab28805245749726eda1a88a/sandbox/processwrapper-sandbox/1372/execroot/cilium && \
   exec env - \
     BAZEL_COMPILER=clang \
     BAZEL_CXXOPTS='-stdlib=libc++' \
     BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a \
     BAZEL_LINKOPTS=-lm:-pthread \
     CC=clang \
     CXX=clang++ \
     CXXFLAGS='-stdlib=libc++' \
     LDFLAGS='-stdlib=libc++' \
     LLVM_CONFIG=/usr/bin/llvm-config \
    PATH=/usr/bin:/root/.local/bin:/root/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
    PWD=/proc/self/cwd \
  /usr/bin/clang '--target=aarch64-unknown-linux-gnu' '-fuse-ld=lld' -U_FORTIFY_SOURCE -fstack-protector -Wall -Wunused-but-set-parameter -Wthread-safety-analysis -Wno-free-nonheap-object -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wno-deprecated-builtins -g0 -O2 '-D_FORTIFY_SOURCE=1' -DNDEBUG -ffunction-sections -fdata-sections '-std=c++0x' -MD -MF bazel-out/aarch64-opt/bin/external/com_google_cel_cpp/common/_objs/json/json.d '-frandom-seed=bazel-out/aarch64-opt/bin/external/com_google_cel_cpp/common/_objs/json/json.o' '-DBAZEL_CURRENT_REPOSITORY="com_google_cel_cpp"' -iquote external/com_google_cel_cpp -iquote bazel-out/aarch64-opt/bin/external/com_google_cel_cpp -iquote external/com_google_absl -iquote bazel-out/aarch64-opt/bin/external/com_google_absl '-DABSL_MIN_LOG_LEVEL=4' -fdebug-types-section -fPIC -Wno-deprecated-declarations '-std=c++20' -fsized-deallocation -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -c external/com_google_cel_cpp/common/json.cc -o bazel-out/aarch64-opt/bin/external/com_google_cel_cpp/common/_objs/json/json.o)
 # Configuration: 2f125838129c3a083292d6497d46584ac8a3a65159ebf628b0573bf0d7d63ead
 # Execution platform: @local_config_platform//:host

Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging
 In file included from external/com_google_cel_cpp/common/json.cc:15:
 In file included from external/com_google_cel_cpp/common/json.h:20:
 In file included from /opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/utility:69:
 In file included from /opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/stl_pair.h:60:
 /opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/type_traits:1326:23: error: incomplete type 'cel::JsonObject' used in type trait expression
  1326 |                     __bool_constant<__has_trivial_destructor(_Tp)>>::type
       |                                     ^
 /opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/type_traits:3297:5: note: in instantiation of template class 'std::is_trivially_destructible<cel::JsonObject>' requested here
  3297 |     is_trivially_destructible<_Tp>::value;
       |     ^
 /opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/variant:367:5: note: in instantiation of variable template specialization 'std::is_trivially_destructible_v<cel::JsonObject>' requested here
   367 |           (is_trivially_destructible_v<_Types> && ...);
       |            ^
 /opt/rh/gcc-toolset-13/root/usr/lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/variant:376:4: note: in instantiation of static data member 'std::__detail::__variant::_Traits<cel::JsonNull, bool, double, absl::Cord, cel::JsonArray, cel::JsonObject>::_S_trivial_dtor' requested here
   376 |           _S_trivial_dtor && _S_trivial_move_ctor
       |

(... MORE AND MORE SIMILAR ERRORS ...)

fatal error: too many errors emitted, stopping now [-ferror-limit=]
20 errors generated.
Target //:cilium-envoy failed to build

Note: similar errors faced when compiling on x86_64 architecture.

Do you know how can I solve this?

Activity

  1. juan131 commented on Apr 1, 2025

    @juan131
    Author

    As an alternative to use clang-devel and llvm-devel system packages, I also tried installing LLVM 17.0.6 at /usr/lib/llvm-17 downloading it from:

    This seems to work for aarch64 architecture but when I compile on x86_64 architecture, I find the issue below:

     /usr/lib/llvm-17/bin/clang: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /usr/lib/llvm-17/bin/clang)
    

    The x86_64 binaries seems to be compiled on Ubuntu 22 and they expect libstdc++.so.6 to include GLIBCXX_3.4.30, but it includes up to GLIBCXX_3.4.29 (retrieved from strings /lib64/libstdc.so.6 | grep GLIBCXX) on RedHat UBI 9.

  2. juan131 commented on Apr 2, 2025

    @juan131
    Author

    I'm also considering building LLVM 17.0.6 from source but I'd like to avoid that if possible.

  3. sayboras commented on Apr 2, 2025

    @sayboras
    Member

    Thanks for your issue, tbh I didn't try to build cilium/proxy from redhat, will give it a try some time next week.

  4. sayboras commented on Apr 28, 2025

    @sayboras
    Member

    Sorry, I didn't have time to try it out, might need some help here.

  5. juan131 commented on Apr 28, 2025

    @juan131
    Author

    FYI @sayboras I sorted it out by compiling LLVM 17.0.6 from source.

    That said, I don't think that's a good solution given it makes it much harder to set up the compilation environment. Ideally, it'd be nice to make the C++ code compatible with modern GCC|G++ versions.

  6. sayboras commented on Apr 28, 2025

    @sayboras
    Member

    Agreed. However, the last time i tried to upgrade gcc|g++ versions, I run into a couple of upstream errors such as envoyproxy/envoy#39164

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions