--- a/third_party/gpus/rocm_configure.bzl 2020-05-23 20:16:39.932384302 +0200 +++ b/third_party/gpus/rocm_configure.bzl 2020-05-23 20:16:26.143505404 +0200 @@ -43,7 +43,7 @@ _DEFAULT_ROCM_VERSION = "" _DEFAULT_MIOPEN_VERSION = "" -_DEFAULT_ROCM_TOOLKIT_PATH = "/opt/rocm" +_DEFAULT_ROCM_TOOLKIT_PATH = "/usr" _DEFAULT_ROCM_AMDGPU_TARGETS = ["gfx803", "gfx900"] def verify_build_defines(params): @@ -182,8 +182,7 @@ inc_dirs = get_cxx_inc_directories(repository_ctx, cc) # Add numpy headers - inc_dirs.append("/usr/lib/python2.7/dist-packages/numpy/core/include") - + inc_dirs.append("/usr/lib64/python3.6/site-packages/numpy/core/include/numpy") return inc_dirs def _rocm_include_path(repository_ctx, rocm_config): @@ -204,51 +203,59 @@ inc_dirs.append(rocm_config.rocm_toolkit_path + "/include") # Add HSA headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hsa/include") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hsa") # Add HIP headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip/hcc_detail") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hip/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip/hcc_detail") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hip/include") # Add HIP-Clang headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/8.0/include") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/9.0.0/include") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/10.0.0/include") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/11.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/8.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/9.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/10.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/llvm/lib/clang/11.0.0/include") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/lib/hip/include/hip") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/lib/hip/include/hip/hcc_detail") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/lib/hip/include") # Add rocrand and hiprand headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/rocrand/include") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hiprand/include") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocrand") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hiprand") # Add rocfft headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/rocfft/include") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocfft") # Add rocBLAS headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/rocblas/include") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocblas") # Add MIOpen headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/miopen/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/miopen/include") # Add RCCL headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/rccl/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/rccl/include") + inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/miopen") # Add hcc headers - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/include") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/7.0.0/include/") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/7.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/7.0.0/include/") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/7.0.0/include") # Newer hcc builds use/are based off of clang 8.0.0. - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/8.0.0/include/") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/8.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/8.0.0/include/") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/8.0.0/include") # Support hcc based off clang 9.0.0, included in ROCm2.2 - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/9.0.0/include/") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/9.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/9.0.0/include/") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/9.0.0/include") # Support hcc based off clang 10.0.0, included in ROCm2.8 - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/10.0.0/include/") - inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/10.0.0/include") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/compiler/lib/clang/10.0.0/include/") + #inc_dirs.append(rocm_config.rocm_toolkit_path + "/hcc/lib/clang/10.0.0/include") + + # hcc 3.0 and clang 11.0.0 + inc_dirs.append("/usr/lib/hcc/3.3/include") + inc_dirs.append("/usr/lib/hcc/3.3/lib/clang/11.0.0/include") return inc_dirs @@ -329,8 +336,8 @@ if get_host_environ(repository_ctx, name): return "True" - # grep for "HIP_COMPILER=clang" in /opt/rocm/hip/lib/.hipInfo - cmd = "grep HIP_COMPILER=clang %s/hip/lib/.hipInfo || true" % rocm_config.rocm_toolkit_path + # grep for "HIP_COMPILER=clang" in /usr/lib/hip/lib/.hipInfo + cmd = "grep HIP_COMPILER=clang %s/lib/hip/lib/.hipInfo || true" % rocm_config.rocm_toolkit_path grep_result = execute(repository_ctx, [bash_bin, "-c", cmd], empty_stdout_fine = True) result = grep_result.stdout.strip() if result == "HIP_COMPILER=clang": @@ -436,13 +443,13 @@ libs_paths = [ (name, _rocm_lib_paths(repository_ctx, name, path)) for name, path in [ - ("hip_hcc", rocm_config.rocm_toolkit_path), - ("rocblas", rocm_config.rocm_toolkit_path + "/rocblas"), - ("rocfft", rocm_config.rocm_toolkit_path + "/rocfft"), - ("hiprand", rocm_config.rocm_toolkit_path + "/hiprand"), - ("MIOpen", rocm_config.rocm_toolkit_path + "/miopen"), - ("rccl", rocm_config.rocm_toolkit_path + "/rccl"), - ("hipsparse", rocm_config.rocm_toolkit_path + "/hipsparse"), + ("hip_hcc", rocm_config.rocm_toolkit_path + "/lib/hip/lib"), + ("rocblas", rocm_config.rocm_toolkit_path + "/lib64"), + ("rocfft", rocm_config.rocm_toolkit_path + "/lib64"), + ("hiprand", rocm_config.rocm_toolkit_path + "/lib64"), + ("MIOpen", rocm_config.rocm_toolkit_path + "/lib64"), + ("rccl", rocm_config.rocm_toolkit_path + "/lib64"), + ("hipsparse", rocm_config.rocm_toolkit_path + "/lib64"), ] ] @@ -608,42 +615,84 @@ copy_rules = [ make_copy_dir_rule( repository_ctx, - name = "rocm-include", - src_dir = rocm_toolkit_path + "/include", - out_dir = "rocm/include", - ), - make_copy_dir_rule( - repository_ctx, + #name = "rocm-include", + #src_dir = rocm_toolkit_path + "/include", name = "rocfft-include", - src_dir = rocm_toolkit_path + "/rocfft/include", + src_dir = rocm_toolkit_path + "/include/rocFFT", out_dir = "rocm/include/rocfft", ), make_copy_dir_rule( repository_ctx, name = "rocblas-include", - src_dir = rocm_toolkit_path + "/rocblas/include", + src_dir = rocm_toolkit_path + "/include/rocblas", out_dir = "rocm/include/rocblas", ), make_copy_dir_rule( repository_ctx, + name = "rocblas-include-link", + src_dir = rocm_toolkit_path + "/include/rocblas", + out_dir = "rocm/include", + ), + make_copy_dir_rule( + repository_ctx, + #name = "rocfft-include", + #src_dir = rocm_toolkit_path + "/rocfft/include", + #out_dir = "rocm/include/rocfft", name = "miopen-include", - src_dir = rocm_toolkit_path + "/miopen/include", + src_dir = rocm_toolkit_path + "/include/miopen", out_dir = "rocm/include/miopen", ), make_copy_dir_rule( repository_ctx, - name = "rccl-include", - src_dir = rocm_toolkit_path + "/rccl/include", - out_dir = "rocm/include/rccl", + #name = "rocblas-include", + #src_dir = rocm_toolkit_path + "/rocblas/include", + #out_dir = "rocm/include/rocblas", + name = "hipsparse-include", + src_dir = rocm_toolkit_path + "/include/hipsparse", + out_dir = "rocm/include/hipsparse", ), make_copy_dir_rule( repository_ctx, - name = "hipsparse-include", - src_dir = rocm_toolkit_path + "/hipsparse/include", - out_dir = "rocm/include/hipsparse", + #name = "miopen-include", + #src_dir = rocm_toolkit_path + "/miopen/include", + #out_dir = "rocm/include/miopen", + name = "hipcub-include", + src_dir = rocm_toolkit_path + "/include/hipcub", + out_dir = "rocm/include/hipcub", + ), + make_copy_dir_rule( + repository_ctx, + name = "hip-include", + src_dir = rocm_toolkit_path + "/lib/hip/include/hip", + out_dir = "rocm/include/hip", ), ] - + # Copy required include files + copy_rules.append(make_copy_files_rule( + repository_ctx, + name = "files", + srcs = [ + "/usr/include/rccl.h", + "/usr/include/rocprim/rocprim_version.hpp", + "/usr/include/hiprand/hiprand.h", + "/usr/include/hiprand/hiprand_version.h", + "/usr/include/hiprand/hiprand_hcc.h", + "/usr/include/rocrand/rocrand.h", + "/usr/include/rocrand/rocrand_discrete_types.h", + "/usr/include/rocrand/rocrand_version.h", + ], + outs = [ + "rocm/include/rccl/rccl.h", + "rocm/include/rocprim/rocprim_version.hpp", + "rocm/include/hiprand/hiprand.h", + "rocm/include/hiprand/hiprand_version.h", + "rocm/include/hiprand/hiprand_hcc.h", + "rocm/include/rocrand/rocrand.h", + "rocm/include/rocrand/rocrand_discrete_types.h", + "rocm/include/rocrand/rocrand_version.h", + ], + )) + rocm_libs = _find_libs(repository_ctx, rocm_config, bash_bin) rocm_lib_srcs = [] rocm_lib_outs = [] @@ -681,11 +730,13 @@ "%{rccl_lib}": rocm_libs["rccl"].file_name, "%{hipsparse_lib}": rocm_libs["hipsparse"].file_name, "%{copy_rules}": "\n".join(copy_rules), - "%{rocm_headers}": ('":rocm-include",\n' + + "%{rocm_headers}": ('":files",\n' + '":rocfft-include",\n' + '":rocblas-include",\n' + '":miopen-include",\n' + - '":rccl-include",\n' + + '":hip-include",\n' + + '":hipcub-include",\n' + + '":rocblas-include-link",\n' + '":hipsparse-include",'), }, ) @@ -702,8 +753,9 @@ rocm_defines["%{host_compiler_prefix}"] = host_compiler_prefix - rocm_defines["%{linker_bin_path}"] = rocm_config.rocm_toolkit_path + "/hcc/compiler/bin" - + #rocm_defines["%{linker_bin_path}"] = rocm_config.rocm_toolkit_path + "/hcc/compiler/bin" + rocm_defines["%{linker_bin_path}"] = rocm_config.rocm_toolkit_path + "/lib/hcc/3.3/bin" + # For gcc, do not canonicalize system header paths; some versions of gcc # pick the shortest possible path for system includes when creating the # .d file - given that includes that are prefixed with "../" multiple @@ -753,14 +805,14 @@ tpl_paths["crosstool:clang/bin/crosstool_wrapper_driver_rocm"], { "%{cpu_compiler}": str(cc), - "%{hipcc_path}": rocm_config.rocm_toolkit_path + "/bin/hipcc", + "%{hipcc_path}": rocm_config.rocm_toolkit_path + "/lib/hcc/3.3/bin/hcc", "%{hipcc_env}": _hipcc_env(repository_ctx), "%{hipcc_is_hipclang}": _hipcc_is_hipclang(repository_ctx, rocm_config, bash_bin), - "%{rocr_runtime_path}": rocm_config.rocm_toolkit_path + "/lib", + "%{rocr_runtime_path}": rocm_config.rocm_toolkit_path + "/lib64", "%{rocr_runtime_library}": "hsa-runtime64", - "%{hip_runtime_path}": rocm_config.rocm_toolkit_path + "/hip/lib", + "%{hip_runtime_path}": rocm_config.rocm_toolkit_path + "/lib/hip/lib", "%{hip_runtime_library}": "hip_hcc", - "%{hcc_runtime_path}": rocm_config.rocm_toolkit_path + "/hcc/lib", + "%{hcc_runtime_path}": rocm_config.rocm_toolkit_path + "/lib/hcc/3.3/lib", "%{hcc_runtime_library}": "mcwamp", "%{crosstool_verbose}": _crosstool_verbose(repository_ctx), "%{gcc_host_compiler_path}": str(cc),