From 87b8d539b5cf7fe5f02813e7951df6716ea99b1c Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Thu, 5 May 2022 12:53:37 +0000 Subject: [PATCH 01/23] Add oneDNN submodule to 3rd_party --- .gitmodules | 3 +++ src/3rd_party/oneDNN | 1 + 2 files changed, 4 insertions(+) create mode 160000 src/3rd_party/oneDNN diff --git a/.gitmodules b/.gitmodules index a1a876d8b..d44a4f73b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,3 +20,6 @@ [submodule "src/3rd_party/simple-websocket-server"] path = src/3rd_party/simple-websocket-server url = https://github.com/marian-nmt/Simple-WebSocket-Server +[submodule "src/3rd_party/oneDNN"] + path = src/3rd_party/oneDNN + url = https://github.com/oneapi-src/oneDNN.git diff --git a/src/3rd_party/oneDNN b/src/3rd_party/oneDNN new file mode 160000 index 000000000..11fa74eaf --- /dev/null +++ b/src/3rd_party/oneDNN @@ -0,0 +1 @@ +Subproject commit 11fa74eaf03af9848c1bb5fffb4cbb2866aadf42 From e72a0dd7d1e7d080513fc100c1c9328f70406a36 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Thu, 5 May 2022 14:26:00 +0000 Subject: [PATCH 02/23] Add oneDNN to CMake --- CMakeLists.txt | 2 +- src/3rd_party/CMakeLists.txt | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dbad75cb5..053e2286f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -506,7 +506,7 @@ endif() # Find BLAS library if(COMPILE_CPU) if(NOT GENERATE_MARIAN_INSTALL_TARGETS) - set(EXT_LIBS ${EXT_LIBS} intgemm) # Enable intgemm when compiling CPU + set(EXT_LIBS ${EXT_LIBS} intgemm dnnl) # Enable intgemm when compiling CPU add_definitions(-DCOMPILE_CPU=1) endif() if(USE_APPLE_ACCELERATE) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index c21868e33..c6a66e2df 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -13,6 +13,11 @@ if(COMPILE_CPU) set(INTGEMM_DONT_BUILD_TESTS ON CACHE BOOL "Disable intgemm tests") add_subdirectory(./intgemm) endif() + + # OneDNN + set(DNNL_BUILD_TESTS OFF CACHE BOOL "Build dnnl tests") + add_subdirectory(./oneDNN) + include_directories(./oneDNN/include) endif(COMPILE_CPU) if(USE_FBGEMM) @@ -169,7 +174,7 @@ if(CUDA_FOUND) endif(COMPILE_AMPERE) # install nccl in ${CMAKE_BINARY_DIR}/local similar to /usr/local linux installation - # Using $(MAKE) instead of $CMAKE_MAKE_PROGRAM in order to make parallelization in NCCL compilation work with make -j16. + # Using $(MAKE) instead of $CMAKE_MAKE_PROGRAM in order to make parallelization in NCCL compilation work with make -j16. # Apparently this does not get properly propagated otherwise and builts with only a single thread/process. ExternalProject_Add(nccl_install SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nccl From d3414470645841b25a2316ae308dbf46fc78fa05 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Thu, 5 May 2022 16:17:20 +0000 Subject: [PATCH 03/23] Don't build DNNL examples --- src/3rd_party/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index c6a66e2df..208324226 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -16,6 +16,7 @@ if(COMPILE_CPU) # OneDNN set(DNNL_BUILD_TESTS OFF CACHE BOOL "Build dnnl tests") + set(DNNL_BUILD_EXAMPLES OFF CACHE BOOL "Build dnnl examples") add_subdirectory(./oneDNN) include_directories(./oneDNN/include) endif(COMPILE_CPU) From 72e0b1a4cab40a175b2147982368f0031adfaca8 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Thu, 5 May 2022 16:17:48 +0000 Subject: [PATCH 04/23] Allow static builds of DNNL --- src/3rd_party/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index 208324226..28fe38961 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -17,6 +17,11 @@ if(COMPILE_CPU) # OneDNN set(DNNL_BUILD_TESTS OFF CACHE BOOL "Build dnnl tests") set(DNNL_BUILD_EXAMPLES OFF CACHE BOOL "Build dnnl examples") + + if(USE_STATIC_LIBS) + set(DNNL_LIBRARY_TYPE "STATIC") + endif(USE_STATIC_LIBS) + add_subdirectory(./oneDNN) include_directories(./oneDNN/include) endif(COMPILE_CPU) From 641a816aaa1bb97e2f92994df17ef8d544537067 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 09:55:24 +0000 Subject: [PATCH 05/23] Remove MKL include from config parser --- src/common/config_parser.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 404b43f1e..eaa20b2f2 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -15,13 +15,9 @@ #include #include -#if MKL_FOUND -#include -#else #if BLAS_FOUND #include #endif -#endif namespace marian { @@ -267,7 +263,7 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "Pool encoder states instead of using cross attention (selects first encoder state, best used with special token)"); cli.add("--transformer-dim-ffn", "Size of position-wise feed-forward network (transformer)", - 2048); + 2048); cli.add("--transformer-decoder-dim-ffn", "Size of position-wise feed-forward network in decoder (transformer). Uses --transformer-dim-ffn if 0.", 0); From 1e8555f34c54a9f204094f121362a9cefcd91ced Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 09:56:32 +0000 Subject: [PATCH 06/23] Remove MKL from microsoft/quicksand --- src/microsoft/quicksand.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 316c66d11..c3f152211 100644 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -1,10 +1,6 @@ #include "quicksand.h" #include "marian.h" -#if MKL_FOUND -#include "mkl.h" -#endif - #include "data/shortlist.h" #include "translator/beam_search.h" #include "translator/scorers.h" @@ -77,10 +73,6 @@ class BeamSearchDecoder : public IBeamSearchDecoder { device_ = New(deviceId); graph_->setDevice(deviceId, device_); -#if MKL_FOUND - mkl_set_num_threads(options_->get("mkl-threads", 1)); -#endif - std::vector models = options_->get>("model"); @@ -124,7 +116,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { QSNBestBatch decode(const QSBatch& qsBatch, size_t maxLength, const std::unordered_set& shortlist) override { - + std::vector lshOpts = options_->get>("output-approx-knn", {}); ABORT_IF(lshOpts.size() != 0 && lshOpts.size() != 2, "--output-approx-knn takes 2 parameters"); ABORT_IF(lshOpts.size() == 2 && shortlist.size() > 0, "LSH and shortlist cannot be used at the same time"); @@ -138,7 +130,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { shortListGen = New(lshOpts[0], lshOpts[1], vocabs_[1]->lemmaSize(), /*abortIfDynamic=*/true); } else { shortListGen = New(shortlist); - } + } for(auto scorer : scorers_) scorer->setShortlistGenerator(shortListGen); } @@ -299,8 +291,8 @@ bool convertModel(std::string inputFile, std::string outputFile, int32_t targetP } Type targetPrecType = (Type) targetPrec; - if (targetPrecType == Type::packed16 - || targetPrecType == Type::packed8avx2 + if (targetPrecType == Type::packed16 + || targetPrecType == Type::packed8avx2 || targetPrecType == Type::packed8avx512 || (targetPrecType == Type::float32 && addLsh)) { // only allow non-conversion to float32 if we also use the LSH graph->packAndSave(outputFile, configStr.str(), targetPrecType); From 2b44b3f082c14f754de0eeee6560f65013e0f451 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 09:59:17 +0000 Subject: [PATCH 07/23] Remove MKL omatcopy from FBGEMM packed_gemm --- src/tensors/cpu/fbgemm/packed_gemm.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/tensors/cpu/fbgemm/packed_gemm.cpp b/src/tensors/cpu/fbgemm/packed_gemm.cpp index dd81d0f7f..3521a37b0 100644 --- a/src/tensors/cpu/fbgemm/packed_gemm.cpp +++ b/src/tensors/cpu/fbgemm/packed_gemm.cpp @@ -38,11 +38,6 @@ #include #endif -#if MKL_FOUND -#include -#include -#endif - using namespace fbgemm; #endif // USE_FBGEMM @@ -71,7 +66,7 @@ static thread_local PackedGemmMatrixFP16 packedPlaceholder(1, 1, 1, 1, 1, 1, 1, // Copied code from fbgemm. It's padding required from some kernel in FBGEMM // Verbatim - 'required by sw pipelined kernels' // https://github.com/marian-nmt/FBGEMM/blob/master/include/fbgemm/FbgemmFP16.h#L109 -const int PACK16_PADDING = 1024; +const int PACK16_PADDING = 1024; // This is a memory space to store auxiliary variables for FBGEMM (e.g. block row, block column, kernel_ncol_blocks and etc.) const int PACK16_SPECIALMEM = 256; @@ -91,7 +86,7 @@ inline float clip(float value, float min, float max) { // will be removed, when FBGEMM api is changed // blocked row-major format address arithmetic // -// Returns the memory address in the packed (block formatted) matrix array of a specific element +// Returns the memory address in the packed (block formatted) matrix array of a specific element // indexed by the original non-packed array. // // @param r_ row index in the original matrix @@ -338,7 +333,7 @@ void fbgemmPacked16Pack(marian::Tensor out, // quantRangeStdDevs: the range to be quantized for the original float data in multiples standard deviation // the default value is 0.0f which means min/max quantization // only a half range of normal int8 which is [-64, 63] used to avoid overflow -// during the accumulation in VPMADDUBSW instruction +// during the accumulation in VPMADDUBSW instruction // https://intel.github.io/mkl-dnn/dev_guide_int8_computations.html // (e.g. 3.f means the original tensor is quantized // from [mean - 3.f * standard deviation, mean + 3.f * standard deviation] to [-64, 63]) @@ -433,7 +428,7 @@ void fbgemmPacked8Pack(marian::Tensor out, // 4. packing const fbgemm::BlockingFactors* params = getBlockingFactors(packType); - + PackBMatrix packedBN( transpose ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, nrow, ncol, quantized, transpose ? nrow : ncol, packedBuf, 1, params); @@ -491,15 +486,9 @@ void fbgemmPacked16Gemm(marian::Tensor C, packedPlaceholder.pmat_ = (fbgemm::float16*)(B->data() + 256); if(bias != nullptr) { -#if MKL_FOUND - for(int i = 0; i < m; ++i) { - mkl_somatcopy('R', 'N', 1, n, 1, bias->data(), n, C->data() + n * i, n); - } -#else for(int i = 0; i < m; ++i) { std::copy(bias->data(), bias->data() + n, C->data() + n * i); } -#endif } #ifdef _OPENMP From bdc02f6b64c4707e38d6d9ba4a268273e52eeecb Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 12:01:40 +0000 Subject: [PATCH 08/23] Remove MKL from GitHub Actions --- .github/workflows/release.yml | 7 ------- .github/workflows/ubuntu.yml | 10 ---------- .github/workflows/windows.yml | 10 ---------- 3 files changed, 27 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a3761e3b..f679cd090 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,13 +74,6 @@ jobs: - name: Install dependencies run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler libboost-system-dev gcc-${{ env.gcc_version }} g++-${{ env.gcc_version }} - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html - - name: Install MKL - run: | - wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 # The script simplifies installation of different versions of CUDA - name: Install CUDA run: ./scripts/ci/install_cuda_ubuntu.sh ${{ env.cuda_version }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 4a0fa6746..6c7ea2d0d 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -69,15 +69,6 @@ jobs: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler libboost-system-dev \ gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html - - name: Install MKL - run: | - wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 - if: matrix.cpu == true - # The script simplifies installation of different versions of CUDA - name: Install CUDA run: ./scripts/ci/install_cuda_ubuntu.sh ${{ matrix.cuda }} @@ -122,4 +113,3 @@ jobs: ./marian-scorer --version ./marian-server --version ./spm_encode --version - diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index ee85f303d..e57fc797b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -7,7 +7,6 @@ on: branches: [ master ] env: - MKL_URL: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip" BOOST_ROOT: "C:/hostedtoolcache/windows/Boost/1.72.0/x86_64" BOOST_URL: "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe" @@ -34,15 +33,6 @@ jobs: with: submodules: recursive - - name: Download MKL - run: | - # Wget retries downloading files and is faster than Invoke-WebRequest - C:\msys64\usr\bin\wget.exe -nv ${{ env.MKL_URL }} -O mkl.zip - Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl - # Set MKLROOT environment variable so that CMake can find MKL - echo "MKLROOT=${{ github.workspace }}\mkl" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - shell: powershell - - name: Install CUDA run: | .\scripts\ci\install_cuda_windows.ps1 "10.2" From 49140fca29e6de28ea64895eb211e8895e56ae87 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 13:13:58 +0000 Subject: [PATCH 09/23] Remove MKL call from 3in4 transpose, reverting to plain copy (on forward) --- src/tensors/cpu/tensor_operators.cpp | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index 1e1adc38b..f840e1742 100755 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -12,10 +12,6 @@ #include "functional/tensor.h" #include "functional/operators.h" -#if MKL_FOUND -#include -#endif - namespace marian { namespace cpu { @@ -244,8 +240,6 @@ void Transpose0213(Tensor out, Tensor in) { } } -// This function is called only when MKL is available. -#if MKL_FOUND // Given a 4D array, transpose (swap) the initial 3 dimensions while keeping the last dimension. // e.g. 1234 --> 2134, 1234 --> 3214 (4 is always kept). // This is an optimized version for swapping first 3 dimensions @@ -297,7 +291,7 @@ void TransposeFirst3In4(Tensor out, Tensor in, const std::vector& vAxis) { float* outRow = out->data() + dst * innermost; if(!add) { - mkl_somatcopy('R', 'N', 1, innermost, 1.0f, inRow, innermost, outRow, innermost); + std::copy(inRow, inRow + innermost, outRow); } else { for(int ii = 0; ii < innermost; ++ii) { outRow[ii] += inRow[ii]; @@ -307,7 +301,6 @@ void TransposeFirst3In4(Tensor out, Tensor in, const std::vector& vAxis) { } } } -#endif // MKL_FOUND inline void transpose4x4_SSE(const float* A, float* B, @@ -390,10 +383,8 @@ void TransposeGeneric(Tensor out, Tensor in, const std::vector& vAxis) { void TransposeND(Tensor out, Tensor in, const std::vector& vAxis) { if(vAxis == std::vector({0, 2, 1, 3})) Transpose0213(out, in); -#if MKL_FOUND else if(vAxis.size() == 4 && vAxis[3] == 3) TransposeFirst3In4(out, in, vAxis); -#endif // MKL_FOUND else if(vAxis == std::vector({1, 0}) && in->shape()[-1] % 16 == 0 && in->shape()[-2] % 16 == 0) Transpose10(out, in); @@ -942,7 +933,7 @@ void CrossEntropyPick(Tensor out, Tensor in, Tensor labelIndices, float labelSmo // This appears to be safe i.e. that i >= 0 && i < cols is known float logsumexp = std::log(sumexp); float ce = logsumexp - sp[i] + max; // -log(p_i) = - logsoftmax(x_i - max) = - (x_i - max) - log(sum_j exp(x_j - max)) - float ls = logsumexp - mean; + float ls = logsumexp - mean; out->data()[j] = (1.f - labelSmoothingAlpha) * ce + labelSmoothingAlpha * ls; } } @@ -977,7 +968,7 @@ void CrossEntropyPickBackward(Tensor out, // cross-entropy for(int i = 0; i < cols; ++i) { float sub = (float)(i == (int)labelIndices->data()[j]); // delta, true if label index and column index match - float dce = std::exp(sp[i] - max) / sumexp - sub + float dce = std::exp(sp[i] - max) / sumexp - sub + labelSmoothingAlpha * (sub - 1.f / (float)cols); so[i] += adj->data()[j] * dce; } @@ -1383,7 +1374,7 @@ void RMSNormalizationGrad(Tensor gradX_, for(size_t i = 0; i < cols; ++i) { float rmsNorm = (yRow[i] - beta[betaStride * i]) / gamma[gammaStride * i]; float gradNorm = cols * adjRow[i] - rmsNorm * sum_adj_r; - gradNorm /= cols * rms; + gradNorm /= cols * rms; gradXRow[i] += gamma[gammaStride * i] * gradNorm; gradGamma[gammaStride * i] += adjRow[i] * rmsNorm; @@ -1412,7 +1403,7 @@ void RMSNormalizationGrad(Tensor gradX_, for(size_t i = 0; i < cols; ++i) { float rmsNorm = yRow[i] / gamma[gammaStride * i]; float gradNorm = cols * adjRow[i] - rmsNorm * sum_adj_r; - gradNorm /= cols * rms; + gradNorm /= cols * rms; gradXRow[i] += gamma[gammaStride * i] * gradNorm; gradGamma[gammaStride * i] += adjRow[i] * rmsNorm; @@ -1535,7 +1526,7 @@ void LSTMCellForward(Tensor out, std::vector inputs) { template void LSTMOutputForwardTyped(Tensor out_, const std::vector& inputs) { int rows = out_->shape().elements() / out_->shape()[-1]; - + int fVecSize = sizeof(FType) / sizeof(float); int cols = out_->shape()[-1] / fVecSize; @@ -1568,7 +1559,7 @@ void LSTMOutputForward(Tensor out, std::vector inputs) { #ifdef __AVX__ if(cols % 8 == 0) LSTMOutputForwardTyped(out, inputs); - else + else #endif if(cols % 4 == 0) LSTMOutputForwardTyped(out, inputs); From 9da225e2b6b3c5c1a3bafe8633654ed0e74f2cd0 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 13:16:30 +0000 Subject: [PATCH 10/23] Replace MKL sgemm with oneDNN sgemm --- src/tensors/cpu/prod_blas.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/tensors/cpu/prod_blas.h b/src/tensors/cpu/prod_blas.h index a591fdd26..bb0d28a88 100644 --- a/src/tensors/cpu/prod_blas.h +++ b/src/tensors/cpu/prod_blas.h @@ -1,5 +1,5 @@ -#if MKL_FOUND -#include +#if DNNL_FOUND +#include #else #if BLAS_FOUND #include @@ -19,6 +19,21 @@ inline void sgemm(bool transA, float beta, float* c, int ldc) { +#if DNNL_FOUND + dnnl::sgemm(transA ? 't' : 'n', + transB ? 't' : 'n', + rows_a, + rows_b, + width, + alpha, + a, + lda, + b, + ldb, + beta, + c, + ldc); +#else #if BLAS_FOUND cblas_sgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, @@ -38,4 +53,5 @@ inline void sgemm(bool transA, transA; transB; rows_a; rows_b; width; alpha; a; lda; b; ldb; beta; c; ldc; // make compiler happy ABORT("Marian must be compiled with a BLAS library"); #endif +#endif } From 348fa95bdb981750cad9585308988b7a16d8e05c Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 14:23:09 +0000 Subject: [PATCH 11/23] Remove MKL from CMake and improve oneDNN CMake --- CMakeLists.txt | 19 +++++++------------ src/3rd_party/CMakeLists.txt | 6 ++++-- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 053e2286f..7135d2052 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,7 @@ option(USE_CCACHE "Use ccache compiler cache (https://ccache.dev)" OFF) option(USE_CUDNN "Use CUDNN library" OFF) option(USE_DOXYGEN "Build documentation with Doxygen" ON) option(USE_FBGEMM "Use FBGEMM" OFF) -option(USE_MKL "Compile with MKL support" ON) +option(USE_DNNL "Compile with oneDNN support" ON) option(USE_MPI "Use MPI library" OFF) option(USE_NCCL "Use NCCL library" ON) option(USE_SENTENCEPIECE "Download and compile SentencePiece" ON) @@ -506,9 +506,14 @@ endif() # Find BLAS library if(COMPILE_CPU) if(NOT GENERATE_MARIAN_INSTALL_TARGETS) - set(EXT_LIBS ${EXT_LIBS} intgemm dnnl) # Enable intgemm when compiling CPU + set(EXT_LIBS ${EXT_LIBS} intgemm) # Enable intgemm when compiling CPU add_definitions(-DCOMPILE_CPU=1) endif() + + if(USE_DNNL) + set(EXT_LIBS ${EXT_LIBS} dnnl) + add_definitions(-DDNNL_FOUND=1) + endif(USE_DNNL) if(USE_APPLE_ACCELERATE) if(NOT APPLE) message(FATAL_ERROR "FATAL ERROR: Apple Accelerate only works on macOS.") @@ -520,15 +525,6 @@ if(COMPILE_CPU) set(EXT_LIBS ${EXT_LIBS} "-framework Accelerate") add_definitions(-DBLAS_FOUND=1) else(USE_APPLE_ACCELERATE) - if(USE_MKL) - find_package(MKL) - endif(USE_MKL) - if(MKL_FOUND) - include_directories(${MKL_INCLUDE_DIR}) - set(EXT_LIBS ${EXT_LIBS} ${MKL_LIBRARIES}) - set(BLAS_FOUND TRUE) - add_definitions(-DBLAS_FOUND=1 -DMKL_FOUND=1) - else(MKL_FOUND) set(BLAS_VENDOR "OpenBLAS") find_package(BLAS) if(BLAS_FOUND) @@ -539,7 +535,6 @@ if(COMPILE_CPU) add_definitions(-DBLAS_FOUND=1) endif(CBLAS_FOUND) endif(BLAS_FOUND) - endif(MKL_FOUND) endif(USE_APPLE_ACCELERATE) endif(COMPILE_CPU) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index 28fe38961..f6386daad 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -13,18 +13,20 @@ if(COMPILE_CPU) set(INTGEMM_DONT_BUILD_TESTS ON CACHE BOOL "Disable intgemm tests") add_subdirectory(./intgemm) endif() +endif(COMPILE_CPU) +if(USE_DNNL) # OneDNN set(DNNL_BUILD_TESTS OFF CACHE BOOL "Build dnnl tests") set(DNNL_BUILD_EXAMPLES OFF CACHE BOOL "Build dnnl examples") if(USE_STATIC_LIBS) - set(DNNL_LIBRARY_TYPE "STATIC") + set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "specifies whether oneDNN library should be SHARED or STATIC" FORCE) endif(USE_STATIC_LIBS) add_subdirectory(./oneDNN) include_directories(./oneDNN/include) -endif(COMPILE_CPU) +endif(USE_DNNL) if(USE_FBGEMM) # @TODO: find out if this is somehow harmful. This is supppressing CMake warnings for CMAKE_SUPPRESS_DEVELOPER_WARNINGS From 6e60006a16b044c71cc2e939cb3b021287c3e24d Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 14:24:42 +0000 Subject: [PATCH 12/23] Remove MKL guards in prod. This also removes sgemm_batched --- src/tensors/cpu/prod.cpp | 149 ++------------------------------------- 1 file changed, 4 insertions(+), 145 deletions(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 8fcca924b..9fb365187 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -7,13 +7,13 @@ #include "tensors/tensor.h" #include "tensors/tensor_allocator.h" -#if MKL_FOUND -#include -#else +#if DNNL_FOUND +#include +#endif #if BLAS_FOUND #include #endif -#endif + #include "integer_common.h" #include "prod_blas.h" @@ -30,7 +30,6 @@ void Prod(marian::Tensor C, bool transB, float beta, float scalar) { -#if BLAS_FOUND float alpha = scalar; int m = A->shape().elements() / A->shape()[-1]; @@ -63,10 +62,6 @@ void Prod(marian::Tensor C, beta, C->data(), ldc); -#else - C; A; B; transA; transB; beta; scalar; - ABORT("You need to compile with MKL in order to use the CPU version"); -#endif } // dummy implementation, computeType doesn't do anything on CPU @@ -90,7 +85,6 @@ void ProdBatched(marian::Tensor C, bool transB, float beta, float scalar) { -#if BLAS_FOUND float alpha = scalar; // determine meta-shape of bdot operation. Essentially treat the last two dimensions as single elements @@ -145,71 +139,6 @@ void ProdBatched(marian::Tensor C, functional::Shape bShapeMetaF = bShapeMeta; functional::Shape cShapeMetaF = cShapeMeta; -#if MKL_FOUND - CBLAS_TRANSPOSE transA_forarr = CblasNoTrans; - CBLAS_TRANSPOSE transB_forarr = CblasNoTrans; - - if(transA) - transA_forarr = CblasTrans; - - if(transB) - transB_forarr = CblasTrans; - - /* cblas_sgemm_batch allows us to group all the small GEMMs that are done in a for loop with sgemm and compute - * them in only one MKL call. For the API documentation refer to - * https://software.intel.com/content/www/us/en/develop/documentation/mkl-developer-reference-c/top/blas-and-sparse-blas-routines/blas-like-extensions/cblas-gemm-batch.html - * The API supports dependencies, where you can specify one "group" of GEMMs to be computed after another. (This controlled by the group_count parameter). - * In our case, the operations are not dependent on one another so we hardcode one group. The rest of the arguments (with the exception of group_size) are - * the same as the ones that cblas_sgemm expects, with the difference that we are supposed to provide an array pointer (One element per group). - * Weirdly enough, we are required to to provide all of the integer arguments as the MKL_INT datatype - */ - - static const constexpr size_t group_count = 1; // We have one group - const std::vector transa_arr(group_count, transA_forarr); - const std::vector transb_arr(group_count, transB_forarr); - const std::vector m_arr(group_count, (MKL_INT)m); - const std::vector n_arr(group_count, (MKL_INT)n); - const std::vector k_arr(group_count, (MKL_INT)k); - const std::vector alpha_arr(group_count, alpha); - const std::vector beta_arr(group_count, beta); - const std::vector lda_arr(group_count, (MKL_INT)lda); - const std::vector ldb_arr(group_count, (MKL_INT)ldb); - const std::vector ldc_arr(group_count, (MKL_INT)ldc); - const std::vector group_size(group_count, (MKL_INT)batchC); // Group size specifies number of GEMM operations per group (Which is batchC) - - std::vector a_array(batchC, nullptr); - std::vector b_array(batchC, nullptr); - std::vector c_array(batchC, nullptr); - - // This loop initializes the array pointers in the same way as the for loop - // in the normal sgemm version a few lines below - functional::Array dims; - for(int i = 0; i < batchC; ++i) { - cShapeMetaF.dims(i, dims); - auto aIndex = aShapeMetaF.bindex(dims); - auto bIndex = bShapeMetaF.bindex(dims); - - a_array[i] = A->data() + aIndex * strideA; - b_array[i] = B->data() + bIndex * strideB; - c_array[i] = C->data() + i * strideC; - } - cblas_sgemm_batch (CblasRowMajor, - &transa_arr[0], - &transb_arr[0], - &m_arr[0], - &n_arr[0], - &k_arr[0], - &alpha_arr[0], - &a_array[0], - &lda_arr[0], - &b_array[0], - &ldb_arr[0], - &beta_arr[0], - &c_array[0], - &ldc_arr[0], - group_count, - &group_size[0]); -#else functional::Array dims; for(size_t i = 0; i < batchC; ++i) { cShapeMetaF.dims(i, dims); @@ -230,11 +159,6 @@ void ProdBatched(marian::Tensor C, C->data() + i * strideC, ldc); } -#endif -#else - C; A; B; transA; transB; beta; scalar; - ABORT("You need to compile with MKL in order to use the CPU version"); -#endif } @@ -246,7 +170,6 @@ void ProdBatchedLegacy(marian::Tensor C, bool transB, float beta, float scalar) { -#if BLAS_FOUND float alpha = scalar; size_t batchA = A->shape().elements() / (A->shape()[-1] * A->shape()[-2]); @@ -274,66 +197,7 @@ void ProdBatchedLegacy(marian::Tensor C, auto strideC = n * m; auto batchC = std::max(batchA, batchB); -#if MKL_FOUND - CBLAS_TRANSPOSE transA_forarr = CblasNoTrans; - CBLAS_TRANSPOSE transB_forarr = CblasNoTrans; - if(transA) - transA_forarr = CblasTrans; - - if(transB) - transB_forarr = CblasTrans; - - /* cblas_sgemm_batch allows us to group all the small GEMMs that are done in a for loop with sgemm and compute - * them in only one MKL call. For the API documentation refer to - * https://software.intel.com/content/www/us/en/develop/documentation/mkl-developer-reference-c/top/blas-and-sparse-blas-routines/blas-like-extensions/cblas-gemm-batch.html - * The API supports dependencies, where you can specify one "group" of GEMMs to be computed after another. (This controlled by the group_count parameter). - * In our case, the operations are not dependent on one another so we hardcode one group. The rest of the arguments (with the exception of group_size) are - * the same as the ones that cblas_sgemm expects, with the difference that we are supposed to provide an array pointer (One element per group). - * Weirdly enough, we are required to to provide all of the integer arguments as the MKL_INT datatype - */ - - static const constexpr size_t group_count = 1; // We have one group - const std::vector transa_arr(group_count, transA_forarr); - const std::vector transb_arr(group_count, transB_forarr); - const std::vector m_arr(group_count, (MKL_INT)m); - const std::vector n_arr(group_count, (MKL_INT)n); - const std::vector k_arr(group_count, (MKL_INT)k); - const std::vector alpha_arr(group_count, alpha); - const std::vector beta_arr(group_count, beta); - const std::vector lda_arr(group_count, (MKL_INT)lda); - const std::vector ldb_arr(group_count, (MKL_INT)ldb); - const std::vector ldc_arr(group_count, (MKL_INT)ldc); - const std::vector group_size(group_count, (MKL_INT)batchC); // Group size specifies number of GEMM operations per group (Which is batchC) - - std::vector a_array(batchC, nullptr); - std::vector b_array(batchC, nullptr); - std::vector c_array(batchC, nullptr); - - // This loop initializes the array pointers in the same way as the for loop - // in the normal sgemm version a few lines below - for(size_t i = 0; i < batchC; ++i) { - a_array[i] = A->data() + (i % batchA) * strideA; - b_array[i] = B->data() + (i % batchB) * strideB; - c_array[i] = C->data() + i * strideC; - } - cblas_sgemm_batch (CblasRowMajor, - &transa_arr[0], - &transb_arr[0], - &m_arr[0], - &n_arr[0], - &k_arr[0], - &alpha_arr[0], - &a_array[0], - &lda_arr[0], - &b_array[0], - &ldb_arr[0], - &beta_arr[0], - &c_array[0], - &ldc_arr[0], - group_count, - &group_size[0]); -#else for(size_t i = 0; i < batchC; ++i) { sgemm(transA, transB, @@ -349,11 +213,6 @@ void ProdBatchedLegacy(marian::Tensor C, C->data() + i * strideC, (int)ldc); } -#endif -#else - C; A; B; transA; transB; beta; scalar; - ABORT("You need to compile with MKL in order to use the CPU version"); -#endif } void ProdWithBias(marian::Tensor C, From 5879c205cd3bd4759e5c8d01c83b3891e79c2234 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 17 May 2022 15:11:21 +0000 Subject: [PATCH 13/23] Use int in loop for ProdBatched --- src/tensors/cpu/prod.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 9fb365187..d140e09df 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -140,7 +140,7 @@ void ProdBatched(marian::Tensor C, functional::Shape cShapeMetaF = cShapeMeta; functional::Array dims; - for(size_t i = 0; i < batchC; ++i) { + for(int i = 0; i < batchC; ++i) { cShapeMetaF.dims(i, dims); auto aIndex = aShapeMetaF.bindex(dims); auto bIndex = bShapeMetaF.bindex(dims); From 586d20cee2953d0a45943fa7a9c1f0c803971089 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Wed, 18 May 2022 08:48:23 +0000 Subject: [PATCH 14/23] oneDNN only use OMP runtime when specified --- src/3rd_party/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index f6386daad..1a69c843c 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -24,6 +24,9 @@ if(USE_DNNL) set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "specifies whether oneDNN library should be SHARED or STATIC" FORCE) endif(USE_STATIC_LIBS) + if(NOT USE_OPENMP) + set(DNNL_CPU_RUNTIME SEQ CACHE INTERNAL "" FORCE) + endif() add_subdirectory(./oneDNN) include_directories(./oneDNN/include) endif(USE_DNNL) From 3a5d3f37c555c23e901dba7abfb4980de9fb99a9 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Mon, 30 May 2022 10:16:10 +0000 Subject: [PATCH 15/23] Move MSVC unicode flags out of global flags --- CMakeLists.txt | 3 ++- src/3rd_party/CMakeLists.txt | 1 + src/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7135d2052..a75604c3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ endif() # Set compilation flags if(MSVC) # These are used in src/CMakeLists.txt on a per-target basis + list(APPEND EXTRA_DEFINITIONS /DUNICODE /D_UNICODE) list(APPEND ALL_WARNINGS /WX; /W4;) # Disabled bogus warnings for CPU intrinsics and Protobuf: @@ -105,7 +106,7 @@ if(MSVC) set(INTRINSICS "/arch:AVX2") # set(INTRINSICS "/arch:AVX512") # /bigobj is necessary for expression_operators.cpp. See https://stackoverflow.com/questions/15110580/penalty-of-the-msvs-compiler-flag-bigobj - set(CMAKE_CXX_FLAGS "/EHsc /DWIN32 /D_WINDOWS /DUNICODE /D_UNICODE /D_CRT_NONSTDC_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /bigobj ${DISABLE_GLOBALLY}") + set(CMAKE_CXX_FLAGS "/EHsc /DWIN32 /D_WINDOWS /D_CRT_NONSTDC_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /bigobj ${DISABLE_GLOBALLY}") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} /MT /O2 ${INTRINSICS} /Zi /MP /GL /DNDEBUG") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} /MTd /Od /Ob0 ${INTRINSICS} /RTC1 /Zi /D_DEBUG") diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index 1a69c843c..be79fb000 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -29,6 +29,7 @@ if(USE_DNNL) endif() add_subdirectory(./oneDNN) include_directories(./oneDNN/include) + endif(USE_DNNL) if(USE_FBGEMM) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e4599c407..d7ca54089 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -130,7 +130,7 @@ set(MARIAN_SOURCES add_library(marian STATIC ${MARIAN_SOURCES}) -target_compile_options(marian PRIVATE ${ALL_WARNINGS}) +target_compile_options(marian PRIVATE ${ALL_WARNINGS} ${EXTRA_DEFINITIONS}) # Generate git_revision.h to reflect current git revision information # [https://stackoverflow.com/questions/1435953/how-can-i-pass-git-sha1-to-compiler-as-definition-using-cmake] From bfe7146d89c54e7b9fd19e993afddb7f370707be Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Thu, 19 May 2022 07:48:25 +0000 Subject: [PATCH 16/23] Disable DNNL JIT Profiling --- src/3rd_party/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index be79fb000..214a30069 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -20,6 +20,7 @@ if(USE_DNNL) set(DNNL_BUILD_TESTS OFF CACHE BOOL "Build dnnl tests") set(DNNL_BUILD_EXAMPLES OFF CACHE BOOL "Build dnnl examples") + set(DNNL_ENABLE_JIT_PROFILING OFF CACHE INTERNAL "" FORCE) if(USE_STATIC_LIBS) set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "specifies whether oneDNN library should be SHARED or STATIC" FORCE) endif(USE_STATIC_LIBS) @@ -27,6 +28,7 @@ if(USE_DNNL) if(NOT USE_OPENMP) set(DNNL_CPU_RUNTIME SEQ CACHE INTERNAL "" FORCE) endif() + add_subdirectory(./oneDNN) include_directories(./oneDNN/include) From e1bba937778d322abb4fe3fff843854806f1ef5a Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Mon, 30 May 2022 19:10:19 +0000 Subject: [PATCH 17/23] Cache Boost --- .github/workflows/windows.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e57fc797b..52b433cea 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -41,9 +41,17 @@ jobs: echo "$env:CUDA_PATH/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append shell: powershell if: matrix.gpu == true + # Cache boost install + - name: Cache Boost + id: cache-boost + uses: actions/cache@v3 + with: + path: ${{ env.BOOST_ROOT }} + key: ${{ runner.os }}-${{ env.BOOST_URL }} # Boost is no longer pre-installed on GitHub-hosted Windows runners - name: Download Boost + if: ${{ steps.cache-boost.outputs.cache-hit != 'true' }} run: | Write-Host "Downloading Boost to ${{ env.BOOST_ROOT }}" C:\msys64\usr\bin\wget.exe -nv "${{ env.BOOST_URL }}" -O "${{ github.workspace }}/boost.exe" From 201321b91b3c6f0dd54039e971634d296f8b6ba5 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Fri, 3 Jun 2022 10:34:53 +0000 Subject: [PATCH 18/23] Clean up after debug build --- .github/workflows/windows.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 52b433cea..8543de1bc 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -68,6 +68,7 @@ jobs: # Windows CUDA builds use USE_NCCL=off due to compilation errors. - name: Build Debug + id: build-debug uses: lukka/run-cmake@v3 with: buildDirectory: ${{ github.workspace }}/build/Debug @@ -93,6 +94,11 @@ jobs: # able to find sometimes. if: matrix.gpu == true + - name: Cleanup Debug + if: steps.build-debug.conclusion == 'success' + working-directory: ${{ github.workspace }}/build/Debug + run: cmake --build . --target clean + # Windows CUDA builds use USE_NCCL=off due to compilation errors - name: Build Release uses: lukka/run-cmake@v3 From 58a1a441aa66e1ffe671712fb3a88373f8692606 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Fri, 3 Jun 2022 12:44:00 +0000 Subject: [PATCH 19/23] Remove FindMKL.cmake --- cmake/FindMKL.cmake | 137 -------------------------------------------- 1 file changed, 137 deletions(-) delete mode 100644 cmake/FindMKL.cmake diff --git a/cmake/FindMKL.cmake b/cmake/FindMKL.cmake deleted file mode 100644 index 093c02f80..000000000 --- a/cmake/FindMKL.cmake +++ /dev/null @@ -1,137 +0,0 @@ -################################################################################ -# -# \file cmake/FindMKL.cmake -# \author J. Bakosi -# \copyright 2012-2015, Jozsef Bakosi, 2016, Los Alamos National Security, LLC. -# \brief Find the Math Kernel Library from Intel -# \date Thu 26 Jan 2017 02:05:50 PM MST -# -################################################################################ - -# Find the Math Kernel Library from Intel -# -# MKL_FOUND - System has MKL -# MKL_INCLUDE_DIRS - MKL include files directories -# MKL_LIBRARIES - The MKL libraries -# MKL_INTERFACE_LIBRARY - MKL interface library -# MKL_SEQUENTIAL_LAYER_LIBRARY - MKL sequential layer library -# MKL_CORE_LIBRARY - MKL core library -# -# The environment variables MKLROOT and INTEL are used to find the library. -# Everything else is ignored. If MKL is found "-DMKL_ILP64" is added to -# CMAKE_C_FLAGS and CMAKE_CXX_FLAGS. -# -# Example usage: -# -# find_package(MKL) -# if(MKL_FOUND) -# target_link_libraries(TARGET ${MKL_LIBRARIES}) -# endif() - -# If already in cache, be silent -if (MKL_INCLUDE_DIRS AND MKL_LIBRARIES AND MKL_INTERFACE_LIBRARY AND - MKL_SEQUENTIAL_LAYER_LIBRARY AND MKL_CORE_LIBRARY) - set (MKL_FIND_QUIETLY TRUE) -endif() - -if(NOT BUILD_SHARED_LIBS) - if (WIN32) - set(INT_LIB "mkl_intel_ilp64.lib") - set(SEQ_LIB "mkl_sequential.lib") - set(THR_LIB "mkl_intel_thread.lib") - set(COR_LIB "mkl_core.lib") - else() - set(INT_LIB "libmkl_intel_ilp64.a") - set(SEQ_LIB "libmkl_sequential.a") - set(THR_LIB "libmkl_intel_thread.a") - set(COR_LIB "libmkl_core.a") - endif() -else() - set(INT_LIB "mkl_intel_ilp64") - set(SEQ_LIB "mkl_sequential") - set(THR_LIB "mkl_intel_thread") - set(COR_LIB "mkl_core") -endif() - -if(MSVC) - set(ProgramFilesx86 "ProgramFiles(x86)") - set(INTEL_ROOT_DEFAULT $ENV{${ProgramFilesx86}}/IntelSWTools/compilers_and_libraries/windows) -else() - set(INTEL_ROOT_DEFAULT "/opt/intel") -endif() -set(INTEL_ROOT ${INTEL_ROOT_DEFAULT} CACHE PATH "Folder contains intel libs") -find_path(MKL_ROOT include/mkl.h PATHS $ENV{MKLROOT} ${INTEL_ROOT}/mkl - DOC "Folder contains MKL") - -find_path(MKL_INCLUDE_DIR NAMES mkl.h HINTS ${MKL_ROOT}/include /usr/include/mkl) - - -find_library(MKL_INTERFACE_LIBRARY - NAMES ${INT_LIB} - PATHS ${MKL_ROOT}/lib - ${MKL_ROOT}/lib/intel64 - ${MKL_ROOT}/lib/intel64_win - ${INTEL_ROOT}/mkl/lib/intel64) - -find_library(MKL_SEQUENTIAL_LAYER_LIBRARY - NAMES ${SEQ_LIB} - PATHS ${MKL_ROOT}/lib - ${MKL_ROOT}/lib/intel64 - ${INTEL_ROOT}/mkl/lib/intel64) - -find_library(MKL_CORE_LIBRARY - NAMES ${COR_LIB} - PATHS ${MKL_ROOT}/lib - ${MKL_ROOT}/lib/intel64 - ${INTEL_ROOT}/mkl/lib/intel64) - -set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) -set(MKL_LIBRARIES ${MKL_INTERFACE_LIBRARY} ${MKL_SEQUENTIAL_LAYER_LIBRARY} ${MKL_CORE_LIBRARY}) - -if(NOT WIN32 AND NOT APPLE) - # Added -Wl block to avoid circular dependencies. - # https://stackoverflow.com/questions/5651869/what-are-the-start-group-and-end-group-command-line-options - # https://software.intel.com/en-us/articles/intel-mkl-link-line-advisor - set(MKL_LIBRARIES -Wl,--start-group ${MKL_LIBRARIES} -Wl,--end-group) -elseif(APPLE) - # MacOS does not support --start-group and --end-group - set(MKL_LIBRARIES -Wl,${MKL_LIBRARIES} -Wl,) -endif() - -# message("1 ${MKL_INCLUDE_DIR}") -# message("2 ${MKL_INTERFACE_LIBRARY}") -# message("3 ${MKL_SEQUENTIAL_LAYER_LIBRARY}") -# message("4 ${MKL_CORE_LIBRARY}") - -if (MKL_INCLUDE_DIR AND - MKL_INTERFACE_LIBRARY AND - MKL_SEQUENTIAL_LAYER_LIBRARY AND - MKL_CORE_LIBRARY) - - - if (NOT DEFINED ENV{CRAY_PRGENVPGI} AND - NOT DEFINED ENV{CRAY_PRGENVGNU} AND - NOT DEFINED ENV{CRAY_PRGENVCRAY} AND - NOT DEFINED ENV{CRAY_PRGENVINTEL} AND - NOT MSVC) - set(ABI "-m64") - endif() - - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMKL_ILP64 ${ABI}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMKL_ILP64 ${ABI}") - -else() - set(MKL_INCLUDE_DIRS "") - set(MKL_LIBRARIES "") - set(MKL_INTERFACE_LIBRARY "") - set(MKL_SEQUENTIAL_LAYER_LIBRARY "") - set(MKL_CORE_LIBRARY "") - -endif() - -# Handle the QUIETLY and REQUIRED arguments and set MKL_FOUND to TRUE if -# all listed variables are TRUE. -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(MKL DEFAULT_MSG MKL_LIBRARIES MKL_INCLUDE_DIRS MKL_INTERFACE_LIBRARY MKL_SEQUENTIAL_LAYER_LIBRARY MKL_CORE_LIBRARY) - -MARK_AS_ADVANCED(MKL_INCLUDE_DIRS MKL_LIBRARIES MKL_INTERFACE_LIBRARY MKL_SEQUENTIAL_LAYER_LIBRARY MKL_CORE_LIBRARY) From 2b88d5446fdf6313f30887dc52ecabfe5c7e44e1 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Fri, 3 Jun 2022 15:45:21 +0100 Subject: [PATCH 20/23] Update Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01aea0251..051ac3729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- oneDNN is used for GEMM on CPU. ### Fixed - Multi-loss casts type to first loss-type before accumulation (aborted before due to missing cast) @@ -29,6 +30,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Faster LSH top-k search on CPU - Updated intgemm to the latest upstream version - Parameters in npz files are no longer implicitly assumed to be row-ordered. Non row-ordered parameters will result in an abort +- MKL is no longer used as a backend for the CPU. ## [1.11.0] - 2022-02-08 From 4e3572da03bb1789cd1dab53499088e6ea718131 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Fri, 3 Jun 2022 16:07:24 +0100 Subject: [PATCH 21/23] Remove MKL from release workflow --- .github/workflows/release.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f679cd090..e8ca5e780 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -149,15 +149,6 @@ jobs: with: submodules: recursive - - name: Download MKL - run: | - C:\msys64\usr\bin\wget.exe -nv https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -O mkl.zip - Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl - # Set the MKLROOT environment variable so that CMake can find MKL. - # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners - echo "MKLROOT=$env:GITHUB_WORKSPACE/mkl" | Out-File -FilePath $env:GITHUB_ENV -Append - shell: pwsh - - name: Install CUDA run: | .\scripts\ci\install_cuda_windows.ps1 '${{ env.cuda_version }}' From 885270016798cd05ac94de5cd61d2e851a20daab Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Fri, 3 Jun 2022 16:07:42 +0100 Subject: [PATCH 22/23] Replace MKL reference in docs --- doc/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/operators.md b/doc/operators.md index 1e7bba96e..68fcc683d 100644 --- a/doc/operators.md +++ b/doc/operators.md @@ -382,7 +382,7 @@ libraries containing device-specific optimisations. These libraries include: - CBLAS / OpenBLAS - FBGEMM - INTGEMM - - MKL + - oneDNN - GPU - CUDA (cuBLAS) From ce859c4cf532638afd5a5157bc3ce69ad2d7a182 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Fri, 3 Jun 2022 16:09:45 +0100 Subject: [PATCH 23/23] Fix comments mentioning MKL --- src/graph/auto_tuner.h | 2 +- src/graph/expression_operators.cpp | 2 +- src/tensors/backend.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/graph/auto_tuner.h b/src/graph/auto_tuner.h index 01f330853..384f0bec9 100644 --- a/src/graph/auto_tuner.h +++ b/src/graph/auto_tuner.h @@ -26,7 +26,7 @@ class AutoTuner : public AutoTunerRecorder { const size_t collectStatMax = 50; UPtr timer_; - // This structure holds a hash key an algorithm function (e.g. int16, packed gemm, mkl gemm) + // This structure holds a hash key an algorithm function (e.g. int16, packed gemm, fp32 gemm) // for a specific operation size // hash: a unique hash key for each operation size // (e.g. m, n, k, transpose A, transpose B, bias size for GEMM) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index ca5e68054..353b91c7e 100644 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -576,7 +576,7 @@ Expr bdot_legacy(Expr a, Expr b, bool transA, bool transB, float scale) { } Expr affineDefault(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { - // general version, MKL, CBlas or CUDA + // general version (see affine for packed-type variants) int rows = a->shape().elements() / a->shape()[-1]; Expr ones = a->graph()->ones({ rows, 1 }); diff --git a/src/tensors/backend.h b/src/tensors/backend.h index 64a28f925..4492236f0 100644 --- a/src/tensors/backend.h +++ b/src/tensors/backend.h @@ -8,7 +8,7 @@ namespace marian { // GEMM type enum typedef enum { Auto = 0, // auto tuning between available GEMMs - Float32 = 1, // MKL based GEMM, fp32 + Float32 = 1, // fp32 based GEMM FbFp16Packed = 10, // FBGEMM based fp16 GEMM with packing FbInt8Packed = 11 // FBGEMM based int8 GEMM with packing } GemmType;