Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.15.1 (#167)

* Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.15.1

Author: nikitaxgusev

CMakeLists.txt

@@ -335,7 +335,7 @@ endif()
 
 set(CCL_MAJOR_VERSION     "2021")
 set(CCL_MINOR_VERSION     "15")
-set(CCL_UPDATE_VERSION    "0")
+set(CCL_UPDATE_VERSION    "1")
 set(CCL_PRODUCT_STATUS    "Gold")
 string(TIMESTAMP CCL_PRODUCT_BUILD_DATE "%Y-%m-%dT %H:%M:%SZ")
 get_vcs_properties("git")

include/oneapi/ccl/config.h

@@ -32,10 +32,10 @@
 
 #define CCL_MAJOR_VERSION           2021
 #define CCL_MINOR_VERSION           15
-#define CCL_UPDATE_VERSION          0
+#define CCL_UPDATE_VERSION          1
 #define CCL_PRODUCT_STATUS     "Gold"
-#define CCL_PRODUCT_BUILD_DATE "2025-03-24T 03:29:36Z"
-#define CCL_PRODUCT_FULL       "Gold-2021.15.0 2025-03-24T 03:29:36Z (master/b382bf7)"
+#define CCL_PRODUCT_BUILD_DATE "2025-05-05T 03:34:33Z"
+#define CCL_PRODUCT_FULL       "Gold-2021.15.1 2025-05-05T 03:34:33Z (master/0b49b8f)"
 
 #if defined(SYCL_LANGUAGE_VERSION) && defined (__INTEL_LLVM_COMPILER)
 #define CCL_ENABLE_SYCL

man/OneCCL.md

@@ -150,31 +150,6 @@ Arguments<br />
 
 
 
-By-default: &quot;1&quot; 
-        
-
-## CCL_ENABLE_OMP_ALLREDUCE
-
-
-Set this environment variable to use openmp threads for synchronous collectives with direct algorithms on host buffers. 
-        
-
-
-Syntax <br />
-CCL_ENABLE_OMP_ALLREDUCE=&quot;&lt;value&gt;&quot;<br />
-<br />
-Arguments<br />
-&quot;&lt;value&gt;&quot; Description<br />
-
- - 0 Does not use openmp threads for allreduce.<br />
-
-
- - 1 Use openmp threads for allreduce (default).<br />
-<br />
-
-
-
-
 By-default: &quot;1&quot; 
 
 
@@ -1246,14 +1221,24 @@ Specify the maximum threshold for the Allreduce Sycl scale-out algorithm.
 Set the threshold in bytes to specify the Sycl scaleout algorithm in the allreduce collective. Default value is 1048576. &quot;&lt;value&gt;&quot;&quot; : &quot;&gt;=0&quot; 
 
 
-## CCL_SYCL_ALLREDUCE_SCALEOUT_DIRECT_THRESHOLD
+## CCL_SYCL_ALLREDUCE_SCALEOUT
 
 
-Specify the maximum threshold for the Allreduce Sycl scale-out direct algorithm. 
+Specify allreduce SYCL scale-out algorithm. 
 
 
 
-Set the threshold in bytes to specify the Sycl scaleout direct algorithm (call MPI_allreduce directly) in the allreduce collective. Default value is 1048576. &quot;&lt;value&gt;&quot;&quot; : &quot;&gt;=0&quot; 
+Set the algorithm string from a list of available algorithms to set a specific algorithm for scale-out phase. ALLREDUCE algorithms
+ - auto Automatic selection. Default vaue.
+
+ - direct Based on MPI_Iallreduce
+
+ - rabenseifner Rabenseifner&#8217;s algorithm
+
+ - ring Reduce_scatter + allgather ring 
+
+
+
         
 
 ## CCL_SYCL_REDUCE_SCATTER_TMP_BUF
@@ -1298,6 +1283,24 @@ Specify the threshold for the Sycl scaleout algorithm in reduce-scatter.
 Set the threshold in bytes to specify the Sycl scaleout algorithm in the reduce-scatter collective. Default value is 4294967296. &quot;&lt;value&gt;&quot;&quot; : &quot;&gt;=0&quot; 
 
 
+## CCL_SYCL_REDUCE_SCATTER_SCALEOUT
+
+
+Specify reduce-scatter SYCL scale-out algorithm. 
+        
+
+
+Set the algorithm string from a list of available algorithms to set a specific algorithm for scale-out phase. REDUCE_SCATTER algorithms
+ - auto Automatic selection. Default vaue.
+
+ - direct Based on MPI_Ireduce_scatter
+
+ - ring Ring algorithm 
+
+
+
+        
+
 
 Experimental OneCCL Environment Variables Functionality of these variables has not been (fully) tested and, therefore, cannot be supported nor guaranteed. 
 

man/doxconfig

@@ -1,5 +1,5 @@
 PROJECT_NAME           = "Intel® oneAPI Collective Communications Library"
-PROJECT_NUMBER         = "2021.15.0"
+PROJECT_NUMBER         = "2021.15.1"
 
 INPUT = ../src/common/env/vars.hpp ../src/common/env/vars_experimental.hpp
 

man/man3/OneCCL.3

@@ -1,4 +1,4 @@
-.TH "OneCCLvars" 3 "Version 2021.15.0" "Intel® oneAPI Collective Communications Library" \" -*- nroff -*-
+.TH "OneCCLvars" 3 "Tue Mar 18 2025" "Version 2021.15.1" "Intel® oneAPI Collective Communications Library" \" -*- nroff -*-
 .ad l
 .nh
 .SH NAME
@@ -54,10 +54,6 @@ OneCCLvars \- OneCCL Environment Variables
 .br
 .RI "Set this environment variable to enable cache model automatically for synchronous collectives with direct algorithms\&. "
 .ti -1c
-.RI "\fBCCL_ENABLE_OMP_ALLREDUCE\fP"
-.br
-.RI "Set this environment variable to use openmp threads for synchronous collectives with direct algorithms on host buffers\&. "
-.ti -1c
 .RI "\fBCCL_ALLGATHER\fP"
 .br
 .RI "Set allgather algorithm\&. "
@@ -603,32 +599,6 @@ Arguments
 
 .br
 
-.PP
-.PP
-By-default: '1' 
-.SS "CCL_ENABLE_OMP_ALLREDUCE"
-
-.PP
-Set this environment variable to use openmp threads for synchronous collectives with direct algorithms on host buffers\&. Syntax 
-.br
-CCL_ENABLE_OMP_ALLREDUCE='<value>'
-.br
-
-.br
-Arguments
-.br
-'<value>' Description
-.br
-.IP "\(bu" 2
-0 Does not use openmp threads for allreduce\&.
-.br
-
-.IP "\(bu" 2
-1 Use openmp threads for allreduce (default)\&.
-.br
-
-.br
-
 .PP
 .PP
 By-default: '1' 
@@ -1081,7 +1051,7 @@ By-default: '536870912'
 .SH "Author"
 .PP 
 Generated automatically by Doxygen for Intel® oneAPI Collective Communications Library from the source code\&.
-.TH "ExpOneCCLvars" 3 "Version 2021.15.0" "Intel® oneAPI Collective Communications Library" \" -*- nroff -*-
+.TH "ExpOneCCLvars" 3 "Tue Mar 18 2025" "Version 2021.15.1" "Intel® oneAPI Collective Communications Library" \" -*- nroff -*-
 .ad l
 .nh
 .SH NAME
@@ -1164,9 +1134,9 @@ ExpOneCCLvars \- Experimental OneCCL Environment Variables
 .br
 .RI "Specify the maximum threshold for the Allreduce Sycl scale-out algorithm\&. "
 .ti -1c
-.RI "\fBCCL_SYCL_ALLREDUCE_SCALEOUT_DIRECT_THRESHOLD\fP"
+.RI "\fBCCL_SYCL_ALLREDUCE_SCALEOUT\fP"
 .br
-.RI "Specify the maximum threshold for the Allreduce Sycl scale-out direct algorithm\&. "
+.RI "Specify allreduce SYCL scale-out algorithm\&. "
 .ti -1c
 .RI "\fBCCL_SYCL_REDUCE_SCATTER_TMP_BUF\fP"
 .br
@@ -1183,6 +1153,10 @@ ExpOneCCLvars \- Experimental OneCCL Environment Variables
 .RI "\fBCCL_SYCL_REDUCE_SCATTER_SCALEOUT_THRESHOLD\fP"
 .br
 .RI "Specify the threshold for the Sycl scaleout algorithm in reduce-scatter\&. "
+.ti -1c
+.RI "\fBCCL_SYCL_REDUCE_SCATTER_SCALEOUT\fP"
+.br
+.RI "Specify reduce-scatter SYCL scale-out algorithm\&. "
 .in -1c
 .SH "Detailed Description"
 .PP 
@@ -1245,10 +1219,20 @@ By-default: '0 (disabled)'
 
 .PP
 Specify the threshold for the medium size algorithm in allreduce\&. Set the threshold in bytes to specify the medium size algorithm in the allreduce collective\&. Default value is 16777216\&. '<value>'' : '>=0" 
-.SS "CCL_SYCL_ALLREDUCE_SCALEOUT_DIRECT_THRESHOLD"
+.SS "CCL_SYCL_ALLREDUCE_SCALEOUT"
 
 .PP
-Specify the maximum threshold for the Allreduce Sycl scale-out direct algorithm\&. Set the threshold in bytes to specify the Sycl scaleout direct algorithm (call MPI_allreduce directly) in the allreduce collective\&. Default value is 1048576\&. '<value>'' : '>=0" 
+Specify allreduce SYCL scale-out algorithm\&. Set the algorithm string from a list of available algorithms to set a specific algorithm for scale-out phase\&. ALLREDUCE algorithms
+.IP "\(bu" 2
+auto Automatic selection\&. Default vaue\&.
+.IP "\(bu" 2
+direct Based on MPI_Iallreduce
+.IP "\(bu" 2
+rabenseifner Rabenseifner’s algorithm
+.IP "\(bu" 2
+ring Reduce_scatter + allgather ring 
+.PP
+
 .SS "CCL_SYCL_ALLREDUCE_SCALEOUT_THRESHOLD"
 
 .PP
@@ -1269,6 +1253,18 @@ By-default: '0 (disabled)'
 
 .PP
 Specify the threshold for the medium size algorithm in reduce_scatter\&. Set the threshold in bytes to specify the medium size algorithm in the reduce_scatter collective\&. Default value is 67108864\&. '<value>'' : '>=0" 
+.SS "CCL_SYCL_REDUCE_SCATTER_SCALEOUT"
+
+.PP
+Specify reduce-scatter SYCL scale-out algorithm\&. Set the algorithm string from a list of available algorithms to set a specific algorithm for scale-out phase\&. REDUCE_SCATTER algorithms
+.IP "\(bu" 2
+auto Automatic selection\&. Default vaue\&.
+.IP "\(bu" 2
+direct Based on MPI_Ireduce_scatter
+.IP "\(bu" 2
+ring Ring algorithm 
+.PP
+
 .SS "CCL_SYCL_REDUCE_SCATTER_SCALEOUT_THRESHOLD"
 
 .PP

src/atl/mpi/atl_mpi_ctx.cpp

@@ -677,17 +677,11 @@ atl_status_t atl_mpi_ctx::check_impi_env(const atl_attr_t& attr) {
 
     if (!getenv("ONEAPI_ROOT") && !getenv("I_MPI_ROOT")) {
         atl_mpi_lib_type_t type = ATL_MPI_LIB_IMPI;
-        LOG_ERROR("CCL/MPI uses ",
-                  mpi_lib_infos[type].version_prefix_1,
-                  " but neither I_MPI_ROOT nor ONEAPI_ROOT is set. ",
-                  "Please source ",
-                  mpi_lib_infos[type].kind_value,
-                  " version of ",
-                  mpi_lib_infos[type].version_prefix_1,
-                  " (",
-                  mpi_lib_infos[type].min_version_value,
-                  " or higher version).");
-        return ATL_STATUS_FAILURE;
+        LOG_INFO("oneCCL MPI network transport layer is using ",
+                 mpi_lib_infos[type].version_prefix_1,
+                 " but $I_MPI_ROOT is not set.",
+                 " Transport variables will be initialized automatically.",
+                 " To override them run `source $I_MPI_ROOT/env/vars.sh`");
     }
 
     return ATL_STATUS_SUCCESS;

src/coll/algorithms/utils/sycl_coll_base.cpp

@@ -506,8 +506,10 @@ size_t get_tmp_buf_size_per_rank() {
 
 std::vector<sycl::event> get_sycl_events(const ccl::vector_class<ccl::event> &deps) {
     std::vector<sycl::event> ret;
-    for (auto &dep : deps) {
-        ret.push_back(dep.get_native());
+    if (!group_impl::is_group_active) {
+        for (auto &dep : deps) {
+            ret.push_back(dep.get_native());
+        }
     }
     return ret;
 }

src/coll/coll_util.cpp

@@ -781,16 +781,16 @@ bool is_reduce_scatter_inplace(const void* send_buf,
 
 // this is needed to enable the direct algo fallback for pt2pt when used inside the Group API
 void enable_direct_fallback_for_pt2pt() {
-    CCL_THROW_IF_NOT(ccl::global_data::env().fallback_recv,
-                     "CCL_RECV global fallback table is null");
-    CCL_THROW_IF_NOT(ccl::global_data::env().fallback_send,
-                     "CCL_SEND global fallback table is null");
-    ccl::global_data::env().store_fallback_recv =
-        std::make_shared<ccl_selection_table_t<ccl_coll_recv_algo>>(
-            *(ccl::global_data::env().fallback_recv));
-    ccl::global_data::env().store_fallback_send =
-        std::make_shared<ccl_selection_table_t<ccl_coll_send_algo>>(
-            *(ccl::global_data::env().fallback_send));
+    if (!ccl::global_data::env().fallback_recv) {
+        ccl::global_data::env().fallback_recv =
+            std::make_shared<ccl_selection_table_t<ccl_coll_recv_algo>>();
+    }
+
+    if (!ccl::global_data::env().fallback_send) {
+        ccl::global_data::env().fallback_send =
+            std::make_shared<ccl_selection_table_t<ccl_coll_send_algo>>();
+    }
+
     ccl_algorithm_selector_base<ccl_coll_recv_algo>::insert(
         *(ccl::global_data::env().fallback_recv),
         0,
@@ -801,6 +801,14 @@ void enable_direct_fallback_for_pt2pt() {
         0,
         CCL_SELECTION_MAX_COLL_SIZE,
         ccl_coll_send_direct);
+
+    ccl::global_data::env().store_fallback_recv =
+        std::make_shared<ccl_selection_table_t<ccl_coll_recv_algo>>(
+            *(ccl::global_data::env().fallback_recv));
+
+    ccl::global_data::env().store_fallback_send =
+        std::make_shared<ccl_selection_table_t<ccl_coll_send_algo>>(
+            *(ccl::global_data::env().fallback_send));
 }
 
 // this is needed to preserve general fallback table for pt2pt when used outside the Group API

src/common/api_wrapper/api_wrapper.cpp

@@ -156,7 +156,9 @@ void print_error(int error, lib_info_t& info) {
                  " - path contains invalid characters");
     }
     else if (error == CCL_LOAD_LB_DLOPEN_ERROR) {
-        LOG_WARN("could not open the library: ", info.path.c_str(), " - ", dlerror());
+        // Log as `info`, because in some cases we have to test multiple paths
+        // and we do not want to write excess information into users screen
+        LOG_INFO("could not open the library: ", info.path.c_str(), " - ", dlerror());
     }
 }