From 5eb22fe8d58ac1b932a2489d2797c13d123d9b8f Mon Sep 17 00:00:00 2001 From: learner0018 <212705726+learner0018@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:44:48 +0530 Subject: [PATCH 1/6] Draft: Add activation function testing logic --- app/test_activations.f90 | 60 ++ src/stdlib_specialfunctions_activations.f90 | 761 ++++++++++++++++++ test/specialfunctions/data/elu_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/elu_input.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/elu_output.npy | Bin 0 -> 8136 bytes .../specialfunctions/data/relu_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/relu_input.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/relu_output.npy | Bin 0 -> 8136 bytes .../specialfunctions/data/selu_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/selu_input.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/selu_output.npy | Bin 0 -> 8136 bytes .../data/sigmoid_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/sigmoid_input.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/sigmoid_output.npy | Bin 0 -> 8136 bytes .../data/softplus_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/softplus_input.npy | Bin 0 -> 8136 bytes .../specialfunctions/data/softplus_output.npy | Bin 0 -> 8136 bytes .../data/softsign_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/softsign_input.npy | Bin 0 -> 8136 bytes .../specialfunctions/data/softsign_output.npy | Bin 0 -> 8136 bytes .../specialfunctions/data/tanh_derivative.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/tanh_input.npy | Bin 0 -> 8136 bytes test/specialfunctions/data/tanh_output.npy | Bin 0 -> 8136 bytes test/specialfunctions/generate_data.py | 41 + 24 files changed, 862 insertions(+) create mode 100644 app/test_activations.f90 create mode 100644 src/stdlib_specialfunctions_activations.f90 create mode 100644 test/specialfunctions/data/elu_derivative.npy create mode 100644 test/specialfunctions/data/elu_input.npy create mode 100644 test/specialfunctions/data/elu_output.npy create mode 100644 test/specialfunctions/data/relu_derivative.npy create mode 100644 test/specialfunctions/data/relu_input.npy create mode 100644 test/specialfunctions/data/relu_output.npy create mode 100644 test/specialfunctions/data/selu_derivative.npy create mode 100644 test/specialfunctions/data/selu_input.npy create mode 100644 test/specialfunctions/data/selu_output.npy create mode 100644 test/specialfunctions/data/sigmoid_derivative.npy create mode 100644 test/specialfunctions/data/sigmoid_input.npy create mode 100644 test/specialfunctions/data/sigmoid_output.npy create mode 100644 test/specialfunctions/data/softplus_derivative.npy create mode 100644 test/specialfunctions/data/softplus_input.npy create mode 100644 test/specialfunctions/data/softplus_output.npy create mode 100644 test/specialfunctions/data/softsign_derivative.npy create mode 100644 test/specialfunctions/data/softsign_input.npy create mode 100644 test/specialfunctions/data/softsign_output.npy create mode 100644 test/specialfunctions/data/tanh_derivative.npy create mode 100644 test/specialfunctions/data/tanh_input.npy create mode 100644 test/specialfunctions/data/tanh_output.npy create mode 100644 test/specialfunctions/generate_data.py diff --git a/app/test_activations.f90 b/app/test_activations.f90 new file mode 100644 index 000000000..5cb5ccb78 --- /dev/null +++ b/app/test_activations.f90 @@ -0,0 +1,60 @@ +program test_activations + use iso_fortran_env, only: dp => real64 + ! We use the PARENT module, not the submodule + use stdlib_specialfunctions, only: sigmoid + implicit none + + real(dp) :: x, y, expected + integer :: pass_count + + print *, "==========================================" + print *, " STARTING ACTIVATION FUNCTION LOGIC TEST " + print *, "==========================================" + + pass_count = 0 + + ! Test 1: Sigmoid(0) should be 0.5 + x = 0.0_dp + expected = 0.5_dp + y = sigmoid(x) + + if (abs(y - expected) < 1.0e-9_dp) then + print *, "[PASS] Sigmoid(0.0) calculated correctly." + pass_count = pass_count + 1 + else + print *, "[FAIL] Sigmoid(0.0). Expected:", expected, "Got:", y + end if + + ! Test 2: Sigmoid(1) should be ~0.731 + x = 1.0_dp + expected = 0.7310585786300049_dp + y = sigmoid(x) + + if (abs(y - expected) < 1.0e-9_dp) then + print *, "[PASS] Sigmoid(1.0) calculated correctly." + pass_count = pass_count + 1 + else + print *, "[FAIL] Sigmoid(1.0). Expected:", expected, "Got:", y + end if + + ! Test 3: Sigmoid(-1) should be ~0.268 + x = -1.0_dp + expected = 0.2689414213699951_dp + y = sigmoid(x) + + if (abs(y - expected) < 1.0e-9_dp) then + print *, "[PASS] Sigmoid(-1.0) calculated correctly." + pass_count = pass_count + 1 + else + print *, "[FAIL] Sigmoid(-1.0). Expected:", expected, "Got:", y + end if + + print *, "==========================================" + if (pass_count == 3) then + print *, "ALL TESTS PASSED! (3/3)" + else + print *, "SOME TESTS FAILED:", pass_count, "/ 3" + error stop 1 + end if + +end program test_activations diff --git a/src/stdlib_specialfunctions_activations.f90 b/src/stdlib_specialfunctions_activations.f90 new file mode 100644 index 000000000..7cd0dfc36 --- /dev/null +++ b/src/stdlib_specialfunctions_activations.f90 @@ -0,0 +1,761 @@ +submodule(stdlib_specialfunctions) stdlib_specialfunctions_activations + use stdlib_intrinsics, only: sum => stdlib_sum + implicit none + + real(sp), parameter :: isqrt2_sp = 1._sp / sqrt(2._sp) + real(dp), parameter :: isqrt2_dp = 1._dp / sqrt(2._dp) + +contains + +!================================================== +! Gaussian +!================================================== +elemental module function gaussian_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = exp(-x**2) +end function + +elemental module function gaussian_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = -2._sp * x * exp(-x**2) +end function + +elemental module function gaussian_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = exp(-x**2) +end function + +elemental module function gaussian_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = -2._dp * x * exp(-x**2) +end function + + +!================================================== +! Exponential Linear Unit +!================================================== +elemental module function elu_sp( x , a ) result ( y ) + real(sp), intent(in) :: x + real(sp), intent(in) :: a + real(sp) :: y + y = merge( x , a * (exp(x) - 1._sp), x >= 0._sp) +end function + +elemental module function elu_grad_sp( x , a ) result ( y ) + real(sp), intent(in) :: x + real(sp), intent(in) :: a + real(sp) :: y + y = merge( 1._sp , a * exp(x), x >= 0._sp) +end function + +elemental module function elu_dp( x , a ) result ( y ) + real(dp), intent(in) :: x + real(dp), intent(in) :: a + real(dp) :: y + y = merge( x , a * (exp(x) - 1._dp), x >= 0._dp) +end function + +elemental module function elu_grad_dp( x , a ) result ( y ) + real(dp), intent(in) :: x + real(dp), intent(in) :: a + real(dp) :: y + y = merge( 1._dp , a * exp(x), x >= 0._dp) +end function + + +!================================================== +! Rectified Linear Unit +!================================================== +elemental module function relu_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = max(0._sp, x) +end function + +elemental module function relu_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = merge( 1._sp , 0._sp, x > 0._sp) +end function + +elemental module function relu_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = max(0._dp, x) +end function + +elemental module function relu_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = merge( 1._dp , 0._dp, x > 0._dp) +end function + + +!================================================== +! leaky Rectified Linear Unit +!================================================== +elemental module function leaky_relu_sp( x , a ) result( y ) + real(sp), intent(in) :: x + real(sp), intent(in) :: a + real(sp) :: y + y = merge( x, a * x , x >= 0._sp) +end function + +elemental module function leaky_relu_grad_sp( x , a ) result( y ) + real(sp), intent(in) :: x + real(sp), intent(in) :: a + real(sp) :: y + y = merge( 1._sp , a , x >= 0._sp) +end function + +elemental module function leaky_relu_dp( x , a ) result( y ) + real(dp), intent(in) :: x + real(dp), intent(in) :: a + real(dp) :: y + y = merge( x, a * x , x >= 0._dp) +end function + +elemental module function leaky_relu_grad_dp( x , a ) result( y ) + real(dp), intent(in) :: x + real(dp), intent(in) :: a + real(dp) :: y + y = merge( 1._dp , a , x >= 0._dp) +end function + + +!================================================== +! GELU: Gaussian Error Linear Units function +!================================================== +elemental module function gelu_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 0.5_sp * x * (1._sp + erf(x * isqrt2_sp)) +end function + +elemental module function gelu_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 0.5_sp * (1._sp + erf(x * isqrt2_sp) ) + y = y + x * isqrt2_sp * exp( - 0.5_sp * x**2 ) +end function + +elemental module function gelu_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 0.5_dp * x * (1._dp + erf(x * isqrt2_dp)) +end function + +elemental module function gelu_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 0.5_dp * (1._dp + erf(x * isqrt2_dp) ) + y = y + x * isqrt2_dp * exp( - 0.5_dp * x**2 ) +end function + + +elemental module function gelu_approx_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 0.5_sp * x * (1._sp + fast_erf(x * isqrt2_sp)) +end function + +elemental module function gelu_approx_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 0.5_sp * (1._sp + fast_erf(x * isqrt2_sp) ) + y = y + x * isqrt2_sp * exp( - 0.5_sp * x**2 ) +end function + +elemental module function gelu_approx_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 0.5_dp * x * (1._dp + fast_erf(x * isqrt2_dp)) +end function + +elemental module function gelu_approx_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 0.5_dp * (1._dp + fast_erf(x * isqrt2_dp) ) + y = y + x * isqrt2_dp * exp( - 0.5_dp * x**2 ) +end function + + +!================================================== +! Scaled Exponential Linear Unit (SELU) +!================================================== +elemental module function selu_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + real(sp), parameter :: scale = 1.0507009873554804934193349852946_sp + real(sp), parameter :: alpha = 1.6732632423543772848170429916717_sp + y = merge( x , alpha * exp(x) - alpha, x > 0._sp) + y = scale * y +end function + +elemental module function selu_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + real(sp), parameter :: scale = 1.0507009873554804934193349852946_sp + real(sp), parameter :: alpha = 1.6732632423543772848170429916717_sp + y = merge( scale , scale * alpha * exp(x), x > 0._sp) +end function + +elemental module function selu_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + real(dp), parameter :: scale = 1.0507009873554804934193349852946_dp + real(dp), parameter :: alpha = 1.6732632423543772848170429916717_dp + y = merge( x , alpha * exp(x) - alpha, x > 0._dp) + y = scale * y +end function + +elemental module function selu_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + real(dp), parameter :: scale = 1.0507009873554804934193349852946_dp + real(dp), parameter :: alpha = 1.6732632423543772848170429916717_dp + y = merge( scale , scale * alpha * exp(x), x > 0._dp) +end function + + +!================================================== +! Sigmoid +!================================================== +elemental module function sigmoid_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 1._sp / (1._sp + exp(-x)) +end function + +elemental module function sigmoid_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = exp(x) / (1._sp + exp(x))**2 +end function + +elemental module function sigmoid_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 1._dp / (1._dp + exp(-x)) +end function + +elemental module function sigmoid_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = exp(x) / (1._dp + exp(x))**2 +end function + + +!================================================== +! SiLU: Sigmoid Linear Unit +!================================================== +elemental module function silu_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = x / (1._sp + exp(-x)) +end function + +elemental module function silu_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = (1._sp + exp(x))**2 + y = exp(x) * ( x + y ) / y +end function + +elemental module function silu_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = x / (1._dp + exp(-x)) +end function + +elemental module function silu_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = (1._dp + exp(x))**2 + y = exp(x) * ( x + y ) / y +end function + + +!================================================== +! Step +!================================================== +elemental module function step_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = merge( 1._sp , 0._sp, x > 0._sp) +end function + +elemental module function step_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 0._sp +end function + +elemental module function step_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = merge( 1._dp , 0._dp, x > 0._dp) +end function + +elemental module function step_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 0._dp +end function + + +!================================================== +! softmax +!================================================== +pure module function softmax_r1_sp( x ) result( y ) + real(sp), intent(in) :: x(:) + real(sp) :: y(size(x)) + + y = exp(x - maxval(x)) + y = y / sum(y) +end function + +pure module function softmax_r2_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:) + real(sp) :: y(size(x,1), size(x,2)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<2)then + do j = 1, size(x,dim=2) + y(:, j) = softmax( x(:, j) ) + end do + else + do j = 1, size(x,dim=1) + y(j, :) = softmax( x(j, :) ) + end do + end if +end function +pure module function softmax_r3_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:,:) + real(sp) :: y(size(x,1), size(x,2), size(x,3)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<3)then + do j = 1, size(x,dim=3) + y(:, :, j) = softmax( x(:, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :) = softmax( x(j, :, :), dim=2 ) + end do + end if +end function +pure module function softmax_r4_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:,:,:) + real(sp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<4)then + do j = 1, size(x,dim=4) + y(:, :, :, j) = softmax( x(:, :, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :, :) = softmax( x(j, :, :, :), dim=3 ) + end do + end if +end function + +pure module function softmax_grad_r1_sp( x ) result( y ) + real(sp), intent(in) :: x(:) + real(sp) :: y(size(x)) + + y = softmax(x) + y = y * (1._sp - y) +end function + +pure module function softmax_grad_r2_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:) + real(sp) :: y(size(x,1), size(x,2)) + + integer, intent(in), optional :: dim + integer :: dim_ + + dim_ = 1; if(present(dim)) dim_ = dim + + y = softmax(x,dim_) + y = y * (1._sp - y) +end function +pure module function softmax_grad_r3_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:,:) + real(sp) :: y(size(x,1), size(x,2), size(x,3)) + + integer, intent(in), optional :: dim + integer :: dim_ + + dim_ = 1; if(present(dim)) dim_ = dim + + y = softmax(x,dim_) + y = y * (1._sp - y) +end function +pure module function softmax_grad_r4_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:,:,:) + real(sp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) + + integer, intent(in), optional :: dim + integer :: dim_ + + dim_ = 1; if(present(dim)) dim_ = dim + + y = softmax(x,dim_) + y = y * (1._sp - y) +end function + +pure module function softmax_r1_dp( x ) result( y ) + real(dp), intent(in) :: x(:) + real(dp) :: y(size(x)) + + y = exp(x - maxval(x)) + y = y / sum(y) +end function + +pure module function softmax_r2_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:) + real(dp) :: y(size(x,1), size(x,2)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<2)then + do j = 1, size(x,dim=2) + y(:, j) = softmax( x(:, j) ) + end do + else + do j = 1, size(x,dim=1) + y(j, :) = softmax( x(j, :) ) + end do + end if +end function +pure module function softmax_r3_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:,:) + real(dp) :: y(size(x,1), size(x,2), size(x,3)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<3)then + do j = 1, size(x,dim=3) + y(:, :, j) = softmax( x(:, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :) = softmax( x(j, :, :), dim=2 ) + end do + end if +end function +pure module function softmax_r4_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:,:,:) + real(dp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<4)then + do j = 1, size(x,dim=4) + y(:, :, :, j) = softmax( x(:, :, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :, :) = softmax( x(j, :, :, :), dim=3 ) + end do + end if +end function + +pure module function softmax_grad_r1_dp( x ) result( y ) + real(dp), intent(in) :: x(:) + real(dp) :: y(size(x)) + + y = softmax(x) + y = y * (1._dp - y) +end function + +pure module function softmax_grad_r2_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:) + real(dp) :: y(size(x,1), size(x,2)) + + integer, intent(in), optional :: dim + integer :: dim_ + + dim_ = 1; if(present(dim)) dim_ = dim + + y = softmax(x,dim_) + y = y * (1._dp - y) +end function +pure module function softmax_grad_r3_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:,:) + real(dp) :: y(size(x,1), size(x,2), size(x,3)) + + integer, intent(in), optional :: dim + integer :: dim_ + + dim_ = 1; if(present(dim)) dim_ = dim + + y = softmax(x,dim_) + y = y * (1._dp - y) +end function +pure module function softmax_grad_r4_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:,:,:) + real(dp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) + + integer, intent(in), optional :: dim + integer :: dim_ + + dim_ = 1; if(present(dim)) dim_ = dim + + y = softmax(x,dim_) + y = y * (1._dp - y) +end function + + +!================================================== +! logsoftmax +!================================================== +pure module function logsoftmax_r1_sp( x ) result( y ) + real(sp), intent(in) :: x(:) + real(sp) :: y(size(x)) + + y = x - maxval(x) + y = y - log( sum(exp(y)) ) +end function + +pure module function logsoftmax_r2_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:) + real(sp) :: y(size(x,1), size(x,2)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<2)then + do j = 1, size(x,dim=2) + y(:, j) = logsoftmax( x(:, j) ) + end do + else + do j = 1, size(x,dim=1) + y(j, :) = logsoftmax( x(j, :) ) + end do + end if +end function +pure module function logsoftmax_r3_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:,:) + real(sp) :: y(size(x,1), size(x,2), size(x,3)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<3)then + do j = 1, size(x,dim=3) + y(:, :, j) = logsoftmax( x(:, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :) = logsoftmax( x(j, :, :), dim=2 ) + end do + end if +end function +pure module function logsoftmax_r4_sp( x , dim ) result( y ) + real(sp), intent(in) :: x(:,:,:,:) + real(sp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<4)then + do j = 1, size(x,dim=4) + y(:, :, :, j) = logsoftmax( x(:, :, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :, :) = logsoftmax( x(j, :, :, :), dim=3 ) + end do + end if +end function + +pure module function logsoftmax_r1_dp( x ) result( y ) + real(dp), intent(in) :: x(:) + real(dp) :: y(size(x)) + + y = x - maxval(x) + y = y - log( sum(exp(y)) ) +end function + +pure module function logsoftmax_r2_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:) + real(dp) :: y(size(x,1), size(x,2)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<2)then + do j = 1, size(x,dim=2) + y(:, j) = logsoftmax( x(:, j) ) + end do + else + do j = 1, size(x,dim=1) + y(j, :) = logsoftmax( x(j, :) ) + end do + end if +end function +pure module function logsoftmax_r3_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:,:) + real(dp) :: y(size(x,1), size(x,2), size(x,3)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<3)then + do j = 1, size(x,dim=3) + y(:, :, j) = logsoftmax( x(:, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :) = logsoftmax( x(j, :, :), dim=2 ) + end do + end if +end function +pure module function logsoftmax_r4_dp( x , dim ) result( y ) + real(dp), intent(in) :: x(:,:,:,:) + real(dp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) + + integer, intent(in), optional :: dim + integer :: dim_, j + + dim_ = 1; if(present(dim)) dim_ = dim + + if(dim_<4)then + do j = 1, size(x,dim=4) + y(:, :, :, j) = logsoftmax( x(:, :, :, j), dim=dim_ ) + end do + else + do j = 1, size(x,dim=1) + y(j, :, :, :) = logsoftmax( x(j, :, :, :), dim=3 ) + end do + end if +end function + + +!================================================== +! softplus +!================================================== +elemental module function softplus_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = log(exp(x) + 1._sp) +end function + +elemental module function softplus_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = exp(x) / (exp(x) + 1._sp) +end function + +elemental module function softplus_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = log(exp(x) + 1._dp) +end function + +elemental module function softplus_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = exp(x) / (exp(x) + 1._dp) +end function + + +!================================================== +! Fast intrinsics for accelerated activations +!================================================== + +! Source: https://fortran-lang.discourse.group/t/fastgpt-faster-than-pytorch-in-300-lines-of-fortran/5385/31 +elemental module function fast_tanh_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + real(sp) :: x2, a, b + + x2 = x*x + a = x * (135135.0_sp + x2 * (17325.0_sp + x2 * (378.0_sp + x2))) + b = 135135.0_sp + x2 * (62370.0_sp + x2 * (3150.0_sp + x2 * 28.0_sp)) + y = merge( a / b , sign(1._sp,x) , x2 <= 25._sp ) +end function + +elemental module function fast_tanh_grad_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + y = 1._sp - fast_tanh(x)**2 +end function + +elemental module function fast_erf_sp( x ) result( y ) + real(sp), intent(in) :: x + real(sp) :: y + real(sp) :: abs_x + + abs_x = abs(x) + y = 1._sp - 1._sp / (1._sp+ 0.278393_sp*abs_x + 0.230389_sp*abs_x**2 + 0.000972_sp*abs_x**3 + 0.078108_sp*abs_x**4)**4 + y = y * sign(1.0_sp,x) +end function + +elemental module function fast_tanh_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + real(dp) :: x2, a, b + + x2 = x*x + a = x * (135135.0_dp + x2 * (17325.0_dp + x2 * (378.0_dp + x2))) + b = 135135.0_dp + x2 * (62370.0_dp + x2 * (3150.0_dp + x2 * 28.0_dp)) + y = merge( a / b , sign(1._dp,x) , x2 <= 25._dp ) +end function + +elemental module function fast_tanh_grad_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + y = 1._dp - fast_tanh(x)**2 +end function + +elemental module function fast_erf_dp( x ) result( y ) + real(dp), intent(in) :: x + real(dp) :: y + real(dp) :: abs_x + + abs_x = abs(x) + y = 1._dp - 1._dp / (1._dp+ 0.278393_dp*abs_x + 0.230389_dp*abs_x**2 + 0.000972_dp*abs_x**3 + 0.078108_dp*abs_x**4)**4 + y = y * sign(1.0_dp,x) +end function + + +end submodule \ No newline at end of file diff --git a/test/specialfunctions/data/elu_derivative.npy b/test/specialfunctions/data/elu_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..f15b52c384e0dfc192f38ea2aef60d06c9f85a9c GIT binary patch literal 8136 zcmeH{=RX&W-^Jz1h{#Sz2wy8ZBXUx*_iB+%S=lQJ*(w<+Ke9zeMnXhZHc`HDW$%iF zkc-lN{q8^EesVwYd2(K#^YFaRc_-?c=;&YCN8&?rlVEG-Y2!gS#YIrGmm`R95$tbx zczIa4THNrkwfnEHYUS){_pk4H&C1R0-(NsNTwFp#P=qUp>;HOGse!gH{d+0E&_!XD zpiYSw#GX7t8zokQ=PXoJ4?xR-?DErjZxy?r-Mi&~BRVDDRp&~ZK0 zrI3ye9CfkEZ*1sryMebMq=pWP^7irDwDibhlS-C!pvSLtd7@+!JtU7$hVMVh0FEY^ zL*)Ss(CYn0dG-?nZs{2)HYqS-I!+@`E`t%T-dlQB>}Nu^c^{R$GZQMOBglq`Oc=gy zDE&y88C#zgGQvukQ489b!ecDZ8gFrBN?^fH4JWcb^21<_-c_CnI*fxi)J-ja9>%IB z?+XJ@R$QQNJJ#(})<_5)Vs9O#O0+i5v}6r39}`ut-@F%a}T>X9!e6eW_q3CzckW8*77|MD0v z8}sw6iE!aZ@niXvU@ka(J-E0+Hag5K(>TK*@%a-YMchF%m8$2-I1OGI(;*?ix)mKZ8;*u>O) z6GM$7Tb`4KIHC^S*~xn%4kv|x&xt?8VUhSuPDx7w`T>gbCAkt%zt(+Z>5l}$NwtMZ z%_R{>L!R?bog_*P?wuOtkivFs~&cS>aMJqahDb){6Y19(fGKpWHoccna4i%S~iHox-p~bUKrr0xDCa z>6_LRAkJ0 zot;&d%6WKwxD=jVaUS;^h5ZM4RPk*huvYh>Dts!6(L&2BJ9j39)QzATA`-!X;P}oQJZFZywOXPNNos?n5o` zM38*_aaEOst&eye@y5NZHwn#am2QfoWt{b&_ z@K*CPJS(OT{&H*cjZl5KmWiCZG^~&Be&)?SVg}HfscbWiGJv5bvm@z@0fM(t3gnVs`lRx$kd)F-dz;i60#3Yc z*%sE&v5gMQZ?FcZ(a+^a$83-%TvHhoY6Axq=Vj{!8+;sOTs6?OMNj`xHIr&vBq;`# z`|{X9i={!JHO>yi718nYf9w!`^_L>0yFE74trl74?6GP*)E#ui0W0phyXu1u@Tzp2 z3e&rWl(Y^;rQT~$`l99?qH`U7Y~})H{ntVM_Il!oi6d4%v0KxOJL2NO!H7TCoWN_( zsMxsagndRUWLG1dG0vCxVu8&Ci$=AU&gCw6AED~Iqv?w3=Y3_l3$AcbQe^duyaAuF zYO-_8ZqS(XNxyN;4FbMhmo@9%P;)?ZafHzwl*}|IPB^>cq+gM+TAMqfA24uB@pyop z{V(6`+aB21T5Nwh;eopEa}&zSo}db@tS9Drf@7&W(V5B%<;(^3JZ&Sm3q3yGis<7(| zgKS+_{Qy5CNw=-eZ~4Kjcah^@fIoUkjRbpt`y+;7JxMk!0L4l7j$fj=2_GYxNu=Dw z6BiQBZ9*XC7pQI~zY0VI(SGu{ZV>LDb_y$>3xeT1+mdeZEwqT%Em3pc#%V6klC6f@ z(02D9Oppi${Y3nbL3l98o!x#keG0~)NiHE+M@E! zPn&TlP@48<(tQB3KlEShvmb!k($&dj>jCP6`1-~5=Jjgrftr^G?AVT0` zK4+bPbw%zPlcofmq;sb*=1IiIKdq~3;fZ+5E1sh8H4$Q)BF6FtNf^*7ey&xUgj;_L zItB4P#Je+Xcm7Uz2qmJuuoG!An#4!?PJ1Q8ql%Kn{Btt6c{t_2S=%=WIWb(l?p#Okm>FVadYsYl(eDu36|lVlh3?k%f)EpTD|z zv#~DC{5m5v8{rM%b(f~IK|0paL8_92+MPqQx%oL*EK*U{q{+oH2b0EcuUx3^{7LB; z&xQC0uLISZd5C0~&;3x7hqm_b6@r3K!KHAkvNiQ7vV(W*%INYD$9<=9D>5I_zu9-5 z?JGc1@K$VpPyqr>>jnLP6`-`Nucs{N8R&F!%fj{*f_RbFgfyxU^rmFNvh+ouAjm37 zJ}CmdozWbLNHMkv%*++_#o)Iat}!+&!Q?^i16yB8&`m3JuPE#}B3uGFZu68PA@=@c zXiF(LmQNPVNR=TmUb5hMWEnQAXLp3g%Fs+=>|iBbj^NDR;mh~RQ54+g%=Nh(>1=DR zbt)AIjlC}K4m0}S3<%rM~Wn+5`61P zjp2X2#6uQQF-fd82o9GHn`D*0u%cig^uZBR( z%YY@}8boByKl)Zs1ARpW5pRK42uxX0eNgxc9}??NX^Yi@@a;lIRaGs#vdRp{&eTEB z)ia>4w+=Mr-r=ak;+txNfh9PQA=h7Q! zj};SKcHW?l#WqmKJo6K=ZpTxW4SBYa zc9h0Gx$1A)0jGkIDAU0X^s*?WH5hk-Rd=VMVW<-yFtfd#j;;UX9{tL`>fm7jz#0&v&j(v+e;D-|CUI zA-soU)jsR<;qURQXPKU6?mea%0$5sAKR}z|oV!=y2dqyPTo9legj@qr>Z|V{ZXD^) zYn>Xz5BB)5W*s7E2e~Ge>xf`JTJzgj_CH%IqbYLdUfpUXgM8uB;TT`3zC$v$CFn2^~U`w43CM;%$1M_@8N zCc>9Bf*(P&XDG!-5kC@gzOQK%Pr0guB232+=wl{?zEtOqiRwA{`-j$ zeYk6FCH5Hwt@M^&#h;PCAR<94@&)@7Hs%$ce}RP+%SS4yuSj?y^f;>aEA&K<)3m58 z;-`PPS;xR4CQRynKDAoHX<=fD%lZ<+Ra-TMW4__e7^UTRo@G3wcDvQtundc(lGIM~ z?;!c#TiMgS9R5vUuLpa3_-_JxJ=p8PUJv$mu*ZQt4(xGYj{|!g*yF$+2mU`f@IQI> Bp(6kQ literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/elu_input.npy b/test/specialfunctions/data/elu_input.npy new file mode 100644 index 0000000000000000000000000000000000000000..f6001e377796ab3f1da30a600262c13f2692f5d0 GIT binary patch literal 8136 zcmbW+&u<)O8Nl&fdu^}RYkPKfW_EUVcIN$2DJjY!1r8Nzc;^IMKq;aUNRdcH6ZL>- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/elu_output.npy b/test/specialfunctions/data/elu_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..a983bb0f8a303d445cb2475466cfc8d0e6b5dcdd GIT binary patch literal 8136 zcmbW)`Cp9d|G@ESp?#UQnYL+`oI)r&;l6EUEuCW*73qYWs6?T%WUr88Uvf^!Qnn&l z-efD$k)%YluQPKKm5@H~ruX+B_|7k{hpCzSdS2IkU3U^aW=)?lPmLGOTkqh%G;~Rb z!$`5isDR-PPGX0E;E=EozrTEfL;RO^z3=816uK1O4-NENwG{vD=;G|`;?&zoyixrB zed+$y$MRWHCkGYlZsIWNj@~^tV|yiDij2no z^QA}S25hNevuh+aKdkSLwb(yBRy+yA4w}8Jbv3s0 z&%RE`b;s&|sxh8`y*s^Q0*n0+5VCX>w*2bH)x)vE!)40`V;xqgPj|*j=2u!dVfO?W z6*^!|=2@TqFz|=v3s({kJra~{PAeE7Ixy`{2A)l zW2t4%9ltnO6A;wehIR4$oYah6kXSxI(be4-|Ea^iPrd)G0$UZc+vPp>@UF0FZ?MnC zR!n+{HUBl}w`W+W%d2{b9apEh=`MEtfngmtu-kjxnsWs^Vd1-k^Vp~edGbHlbn&o* z$FRFS)t4N=zMHsDHx4^LRJJD;TfO9K?MAHkMOUkEtYd*wufMR2Yg^9%Y`*Zxy@lAI zY4@kj!j{d_PMwAgaJ^G89(y*VrG6y#S5D3Y7i`2-w-t`qVf`bjMcBpH|8z3P{>?WX zr-#j*x7J<_d)<5YtJV$i4lR2eW+A zv13l<#iU?or>*lpj^*69>g>gOZoBZ;7VNceN&CaG!J90?mSMvNUFUma4F(98dtiTT z{J5UQK2jc=I1uZ_l{}VV^8&1Xm|@Swzpv_nZF!a$)%p{?n=a+nVy86kdiVx=S7-O~ zCs>aWyt^6LN0W!#O2!uU4_cUrUHP``+$OA}PS(L-Y=(`4<3g-&#n|~%u>1UM`wqj- zzLtGhhOGeS(?-~zLho@uesHkrR{zLa?5x7=ZUxvWhg{Fx#_srGa^MWMU{juP0(S6E z3(Ixb<1>z3@Wu9aZO)#AbvHWivd^9HZc$)Mq^L<_P=6>JsPlYni{sKAa81AI|mmlu3X5* zx-HiT_y>C{dQIdetgl^4wGVbfai7zpvBoRcU$Vk}i0+}$_KkxhPSVq_uy(0}EveWy zt->?gv1)TBYWQJA1}R5IVEa}L^XFsVKYX*Oz75Cg;T3xio3dV^TcNC$x64yK5`#?sIi5Ez?505 zZ(w^Je>!>-_Ed>f*uuc`3xly6`dnY!A3G-Iia_3s=kK=SH7VG8qg}(7VBZ!!g#Y`x2sF=Me?o&7q#e!}~q=7!5Tta0V7!a3LnTPMHkiG3-vX45%5#|PhP z_rZ=`vAI(RyQwT?(lsUSBMX=3bFl#fKF4$@aR0qaSeS%89=>JnSZt~`Kd@Ym=ehgT z%^R_I68R^D*uHYZw5v@V45-Y!FbVr`i^q<#Mtp9JEp}anH7bx~@vx^g9=63daB%aa zS%Ve!qT}Gr$@RD%SJiIz#ZLQMwEjvRJ|Bk39}L6_GIvE?tHt^Ir}*CiSl5F=_7`e! z-m6bEOR!)2M(_?*<9#N7$mn2GjlQ2=SB1}!plQOIN)9efJeBByjkvW#Gou36*Tr*z z5S#t5Zu`1&T%WyPBZ@w9FvR4ccmUR9#grAXA2?9_+j~+;8Qy0#XU;fajnAj74t>wT z$I`n;Zok9xwVkiW!x}oSy2h5``9GN5u<XtWmOTLn{(ac54`4JTaLSWdLaj&87)QrE8G_ojfd3~aB#vPET{D)2c|kX zJ2bGj7t4I~{^Q_KsLepbd=8d6KB?1tf%n0@)TNrw@q8A%+1vIE@7LLfc$b42V+MXZa0fph?>l4oZQRfC>rTABh5Krg!R=+49L${D zx~1&~2X#j$zuK6Ak#kL#vhz;&9wam?Nnd_J958a+RYKToN-_bZu$%m?;1j{oBG&^ofs{WN}m zQkLDKQyi2u-≀1fCz$XT}8`!~1O9)D`|m@cv!0=&9EsJlFplx?=F(_?%1fx6@DL zAawM+izR#U9Np(vbR-_nElJX1&pmjbYH11DcjEd#^U6559iLC;oxdDn(LZ-e`lZe2 z8`0~4eGER&mCoJ?8#&Nkbu&jEiR&Ggd|=c%{M_rPh^RH_T;x08S_sbLYA&ca7|(-W zv+EQ=`1eBVnBQBL;q!IvBOgToo?9iW?-wn>`!s5(EX^06Lx&d)315i&%4cL-5=j7=6&kg@fNN2dNM0hwIzdq2!YzUgwft%3*sB zHdYSd^%3K9H##gR*&6S6&(Yyp0$eAv)5*V^;JA7-8)6MO=ut8}`jj@FACdO`k~HwX z_?@S*{g)C(NqVjSqg@HF)(ki3@l^?LZ;1LIQ!2rwTJCAypoArcr4=DnO5ksCo^icQ z30E(T(R@>^1Yxqfe`$ddoI^qjAx{YtcyKlLu@c5+G%e^0`1yN(^}Uj*glRnAaMNo_ zcyTQ-Vdh08l$^Z!I3!sKTQv&&SDsLUhfQYs_=8Glbx&&l8n1-97j2<2+m+Cd_3*8U zRzllawoSTD35WamuN=Ng2`y%WzSsvUVZiN{^g16Ugl46dM$b{gsSBOk8QoC4IAZAqG6t$_My^TN=#3ivd&HhE^g0-6Q{H2=s}z@Cul0ZwyerEv}l(8@_W|3;{Q=b-0PsH1?} z9_LqW{w@cuug3JsMmdBZUN`@9sT^JgN!wDN$zj{{lyw$aa#)~l?$rCd99}h@pH+QG z4zByJI=gL?!-vUU+_W$`c-GC!YW0)Dh-kspv7T}`>LhOHJ4z0rC6d3cJIdi!N7~po zW^$PAue)GBPYy*dKA){=Y66e=Keh7TGy$i#{*TEInjp^3c+RYg`2LJDuebwEaJpK! z-Xf|ABC>2p2-^EMnJB@lh*+4jw@xN_w_LE-swIrH|wFfZn10ZzIqrrUKDAuq8>sV zJ=)60*TbkNlgs7S^$@@I=??yvI#}d#barrV9TeJI7qy(M1Fa?b7m`Bj;BNe_KUcWd zLGTW4tCv+BT)1!P?;)>+Jo`?Mg?DRV^AYW}5plJU(3<`Agm*2JhA(>hxKAx?OX&N$ zsjUX8#pboTk7{7@7eT6RLJjl`xp`MKuLj!x4&JLRt%2~*vQ0$|)ex1xJ3snLHB6mx z?1)-;HTdsnsGd2p8d`!)4}^A9fjr%M#T=-DoGDKYzDHMqBDM9`Or{E&bAFui>!<{a zMCHE2~#FKRRO77a^szu6_EX?V0pQA z1^B$zJF@j z%4>fa3~eoRoLK%ICd*FiCiQs_g?TkQJy*YjhTeNGvFW9deSf)y@0YhAFBEGX?f(|S zPY$13y0ipZOf}Z)A1Mag)|hbXoHvlzb9Bw;<|61Vvi=fbSp-!|>jfuVUxRR%`O=B= z3L(*PQPupAR}h#J?EGX~0n}?nI0YYk3A?U5UlTd*Kk!pzM)&u40sel|T%UU80e7>v zzp;BRY>Pa5?)9LjaMNYhsWS7&@YpRh?~6PeWWtegA2RQQo>#ANIn8&#{OoL(MDI*^ zmsfWFOxkrQatP_vDM*D3w@zW8X);)bjobCy?Evup?`4+eUJ-eK;hA?enc>;RHhE>H zF)^z5)0vnfU9Vg8{ym2Gd!{J1=>h%x5yLC~chzX!r_6oT@8vSAcS3fH<8$Ud{+3r} zK9ht$E&lfaV_is*Za8Q-p7^D1GIRNwcODUa+LNGT&$ zeeZiFrt9lgmC^TqV7eLQbe(c$MuqAF%qP|LDww@E8t;Tk##wdUDyF<^xW!d;{c1+s zHB3I_3(_idkA6Geb45dgfEta9iqWd<_iSHJs6nG|omQrt9ls zn;3kmuJIjOyzgzcTAo>-&|)_m$?Mg~r)J z^U=ZxRpV`C0#){#bljA!% zzLVnzIew7i2RVL{<0m z-gs=XYJGUDYI&dY*zR@WvEBER8r!{IYHatqtFgyb>!!v&Qr+)rtXy@QsIjWAGgD`~ z?;~}#d!E(V?)z4q?VfLSwtF9_v-+xe*I);$)>or@yLoG{TUG0 zLz5hucC&X^seIuDIm)nmfTm{lDc0mkGr zrt2G%)0oC#%;u{$k}+GUI=+qBR@KHc?%o!Ye0-V*K8=%4^TB7gsrCV%J*wJo zd{%XQrSVzSV<4MPn@yTIA4ij>iki&!=Cgd<7hY2}M$YDYb6LOf4!-O0rhBE!IRxYokV8NY0XYQZ5RgMi4k7J#Av;R7 zKZLaJg|sh)zA%_h)Y{+3l4jXdVki&)?Hsr7&hb=j5 z$ze+lTXNWv!7Sp~JlV42xS4{IDrhP1?`4H277Sp*Qrg4ku{1DUl#dMB{ z$sr+!gd7rbNXQ`}hlCswa!AM_A%}z<5^_k$At8r^91?O!$RQzzgd7rbNXQ`}hlCsw za!AM_A%~P4QgTSiAti^D98z*f$sr|&lpIoWNXa23hm;&ra!AP`C5Mz8QgTSiAti^D z98z*f$sr|&lpHd0$jBihhm0IDa>&RbBZrI}GIGesAtQ&395Qmq$RQ(#j2tp@$jBih Vhm0IDa>&RbBZrI}GIH3n{{v3PIUE20 literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/relu_derivative.npy b/test/specialfunctions/data/relu_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..13e7f0582ce78c0441eaef1b4f942dd4c4502d81 GIT binary patch literal 8136 zcmeIyF$#k)6oBD$^%U7&g$%ltQt0NUxHvdTsF4n(N>aLrr|_a4p^7)?P&)isLNbMq zmwzs*ysQuEtww2GxOq8ixpXHnTDso9ynQykZ{2pE*++2eeYo4ft>- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/relu_output.npy b/test/specialfunctions/data/relu_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..aab040e5a135e0bc62f42e1d67fbfdc8875b1b44 GIT binary patch literal 8136 zcmeI#&ubiI9LMq5WRq+*n`Cx&W_EUVcIKHGAyIlr(L<5e=Q*htr4c;{0b6WB4`M?q zB2^&=E8Z&TLB&I%J$MQpJj9FCQV^k(ib|?jNn2}cTWi|8X zd|#huzTe$^^!!WD?R>dpylK2Wwr6_(t8-&dim~l8+s3wtv6+2yZ_Mp}{gr)ld#1D3 zpWgl2{^{=P`}gjio$mfVzI9?^>y}5ihB6)cuF4b{&nfdX}LU^-S>l3k7w79-2an2 zwm#my<7d`clDo3EO`eg5^?JX^T5rJ2QSU#I@-a#t4D)IT^@ z*k?t4p>J4`n|i+$`K)%`ke}#WZs5DY{x@;nq>h_%oA$2C4|F}N@{D$`O1<3pntV&2 z?;7>3k;6K5u9L^Q^mM%&a<9&1L+Th7H)KWUbBp?Kk<%?4n>aRcY~t9&aT~{N9Jg`Y z!Ep!29UOOY+{JMh$2}bPaNNUjAIE(h_i;SH@c_pI91n3k#PJY^q4K;8^`eFw>K&b% zp^obFF_b>ulZMLA$x!+8R8skQl~jK2CH1*Jw~{)cpZAiwrXQ1%(#zRBDu0f8RGxE> z%AeaFmFL}~@;>NMHJy8}dP1LXZ{BWGz3M%E&b=ysUVBx3-n}aCk1~!jjxvrijxvri zjxvru9DO+YaP;Bm!_kMMA4flqejNQc`f>E*7{D=rV*tkhjsYA4I4U?QI4U?QI4U?Q zI4U>>aSY-Z#4(6t5XT^nDvm0SDvm0SDvm0SAsjO|_vL&&=DRTE}0fzB)P7sk2TVb@jgP zgSz@m_gh`*@wHG_`eWdDo%-wKRL5cAuy9y7EF2aN3x|cn!eQaCa9B7j92SlSjs}hf zjs}hfjs}hfjs}hfjs}hfjs^}JhmFI=VdJoI*f?w)HVzwyjl;%a;CZQdyl^KaCkU893BpjdEn8n9&^E?Z(BH8I9fPb zI9i-Xi+*mY{Ch-;zHTu;S~yxbS~z?hJ`Nv;kHg2|;YZ=v#qb(7%El1br;X zL(tEHxgn@qFh2zK3+9Nx5#xw)#5iIcF^(8Vj3dSoE&hBsdZr362Cuf+N9^;7D*JI1(HQjs!=7 zBf*j2NN}V$QXDCc6i139#gXDjailm>94U?zM~WlGk>W^kq&QL>DUK9JiX+95;z)6% II37{|0yK)QTL1t6 literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/selu_derivative.npy b/test/specialfunctions/data/selu_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..14967b85ff79d9ae0d9bae72572e013588fa1242 GIT binary patch literal 8136 zcmeI!=RX#1*amPTMT3+O$qI?&Ms^)SNJ@58_Q;CtY{~9tMcI|?$c|81xy{I4g~%#d z$%s5X|HAX;dEe$Mmd^*PS(bv?SOb>*5aDTxb-2alv+|ez==IVL*#6XNb=Wi+CpZYxULt=3s`>-*?Ja@|p3A zM>OW%`C}-J2v-j6IED=KAI9NkEJ(5Y$QrxF0(PpDL$BhF8JqS^4d-5^X&f>l1JUJ(Ywb2-1L`+>$=GqWOpvS{&&PE~-|2p+_s96NhtL@26 zM9<j7{2w6oZxkG)JvCyGer^)Z1!Q5@5^TgurG#VxlqE5Ey9pj=yVem5(Ije7;& zcg)2hqo#ExWkMX((UXpPW)d*&l<+8+mcZd!weq|>l3;!JZros167BxGx4Q33;Xcn{ z6=qs#L^x*~Ur3e4+3`shZP9ZG6D4NeZaas4)J1N=7Uyx_?x606T|j*Btnrz2!0cOt zpY;hDI1`h-CtYO_dDdQpwoL{vL}qXN;=BNRuFk zd&DYA7JrqEqwN=E5oqk0WuYMl`$%3x-7+~eTDz~PvdE)I-F84LKpvHxB*o72@~H4u zG_1IB5jELis{+jz;Vfp|_(f6yehMpr?tx1dR^DH5Gbgh-~UvpG{W&m$!8Be(uZY=#lD>Exrs! z8NNsUvML}o`@)~yt%8+($uYH-SFksQj%I}PDrhXH&rl~{h1=StApv+(1kIXfsuD)IvkGTfjuHopk*=v}t+0J8=RfAVYgqTsH8fLQ8H0*w>L4xn9rly8E z!oD=TJzAuW6}2<+O@}nV6vX$@##sXT#AE5z@MNX!XGQ(0g5UdF9>_m)C=^+_610t$J{|_?ab5`vz!_&x}({ z+(3qBa+j>rO}G?^6G zy@sgYJ#*jTmLV#+*sbY`3{k=yKx#v2g!g`tEcXqK(D=fw-k`z=tm}-YsOgQdVKPzp z*wGkH*M{cFx{Tq|8kl`k!~`f>>&<&&0=4aq7bIII2z6AlH!w9tV6;qUcbh3DIdvaf zOPHbP>b@F*OfyW|G)M3l)FWUIjXx42>;qL$Jz9r>>(!$a8V}jJN457Q!K9a z-z{&$>3fOq>FL{Gh$RSY-nPUA)$}dXAC_P}xozU?UJ&ZqxW|aDm&PakWQ9n+krT~ z>QcdN5Aw`4lMC+l;NMrI=+^+QsQM4D3k7k4`x{p)jEv% z{Ci=@DR||4LBeAANA`&?i~~N8(#-iH>wKMeoV*`IRGV79CHvw02H70Rx*uXaK8h}D z-A7B~uIp;+Svc5dqa8Tuj(ZdP59CIWZ-y zL^c>cVA;`|x-Kee&L8;;9o(;uNwCNZem}n2hH~o;6hk9YqiSt+{ zn+}6ZWxBtn+e5JS8reqf`wtNnuJs?D{|613tXXoYaD4KUiPGx}NAda6iUF4ha42$q zY1|))9~8Y$n+ziHx7#$;wImXZxv!&!_D11ncc{W&<0z2z8WN|gqQG##>{1$IG$?3| z36d_+IR5K3T}e+g`h3(Yn8hAp+rOCEBjFJQG=7>5Z$Cm)P}#;Mix_D0JKg=*5rZct zs&P)zu|TetvtmvxCO^k2sxv(X_oia-vxkpS`*;2OInpOMl2>aY>h=USx$egwFFrxy z*q;zf+c=C}w9tI}Jq`+LcWqN_;*rT*x`$~Y9)ESH6VACLz*l8slzuw_(Lc6FibE3- zCC)m|aWn}FR7$=MIY|(>+aFhWJ{g5=95?2BlHt?vM^4H<1u0jqv85k)io)JH>T|E2 zg4X3_H5KnOJmn61n&kNmVn#DXBcGlje`cX{=;U*pA}&Y9`#pysQB#j&_&GAloppS~ zQxWEpv-Re2Dh7rGmxq3*!sX5BgY(y4;PLb;gRQC;h@ma-@C0k(qTA1@>TLe2BxYMZm5@JfM}<*=Phq0lCP7_1wPM2)6J@D zRm@qq5GwN8GA;|Wr#_D*GGwFAQE<>8J{w)+6f?CfIgk>YP3la`L4H(NbCSSI(5o?r zpQ(HaR?T{qc;#HMxO2Xq9?k_vYU2lfhdj97eP8di_Z9Y1v0d&+e+8qi%nPdW`EZHR z8z-I2$Iazf1J8>E;K=H~nHpaJ=kPm09`gmbePk^Apkg6{tOcvAo)zL^^1F%XKZT%j z#J%Y2ML2LWZ`+})2u_D&&zxs0Mw7Pd;+A_c?5ZhEsz!=&Y;ddFU!erMjLO^e^GmQQ zK)LriQz;IY$$Ps8mEyjTjn1dvrJzVq+O)X+8YE!})XKxJF>+Sf#$UY*{mvCqS6;Pp{d-|rU0)AOqH(m5A z!G8BpEXj{bEV`#RntN7Z;B!EA8O>X;nZAf7%Xo`P@}*R5yo09pR$ATAJB*4vNY?du zk3J1*2RfQ+#PQf4VmGeFWW_)5#G)2W;N)aVdQamUxUdlL!u;e zEzFhf(SC8O#h+rF(O!*R z^-wc1q4LHDh}mo=80LL|-Qpfc50(a8`D?U)FrooA2QRV&?;?W0;uRO=LBv4ovsYv* zL=;e|wNKeM;wiaL&XxH_P&JnmF4#9gC)qQ;eYpvh%#ZYTyEmiN+Fp2>tOZLUNxW&{ zEvOyl2;*UCMUE)dLfETTq*#5F8kKEBN#a6^@L(G{?s&w-I(-DWQp)O)L+yyKQvO_D z)Q)3CbNae`9Y{|wBCL6Lptgm>MX|dB`gw^ns(haiW?<^f^xzY28t@YezI;O3c$Nv1 zR41P94fZ#V?}WtSZSm;!PO!E0-zdJ(1)dXH8}0AAkSnR7xy03t$ZnO96XD%xHR)7v zSnGyKO{(oT(;kF9Y`hcF)q~zEZ&NJgd(q+%{V=Mm7jr&rMtlcCUr?D=L^o&5@v}?BWRio zeHk@70unwInhwQL(2<`$LY6d&?zp7Dsl`!zCzziJxip6Tnw^vjsbi?v6)nm2cMM}q zub72ye1(y}k1$)sS4fDZO0F}FgJ))gndmhR*SNBkM_Z4O^8r)sw6zQNvBY8a^@!)t9-Au8AgI|Tm$i8YvRaaT-m0^BA3pFdqHh+Y3EJw0#@|5^ zTHE+{{yXw0i&F3T{(xD)o0lVp=WvsD@egw5@RQbeJzMT4$QK3mPLKXX%gTPPL7#c( zijn9U(k?(Fw{FnQbOE2-|I{pELyIZx6U!1J z2gTIydo1B+CTAeS&=Tb1;;0g(mvNYUp`ZTQGTI*SoQS3Pg;AOxs%*BuaNH#4cWv)4 zY>=eS1j()-x1L7Gy3_2-!{d=r`N><^o~O5aZQ!ZC^x=-(e^5MWNgf;h z2Z9SS4gBnz5TecQec8USQ+MuRCkH!wu;ajv13M1vII!cujsrUm>^QLFz>Whu4*Wkl F@IOSwAQk`s literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/selu_input.npy b/test/specialfunctions/data/selu_input.npy new file mode 100644 index 0000000000000000000000000000000000000000..f6001e377796ab3f1da30a600262c13f2692f5d0 GIT binary patch literal 8136 zcmbW+&u<)O8Nl&fdu^}RYkPKfW_EUVcIN$2DJjY!1r8Nzc;^IMKq;aUNRdcH6ZL>- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/selu_output.npy b/test/specialfunctions/data/selu_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..6aec3d35ed64b618a7b4269a2691d99b40f051c9 GIT binary patch literal 8136 zcmbW(`9D&mH?HzoZei4jb`yvzhQlrhPHmi9!qc3Da9VL6JA_`4Hvz$?K5^=PfEl`Y@%{r z%m?gh|FYOZ>>xiS`YpEe0WUNkYw4BZnT!2eyJgNB?2G~#sch`JYi=K3Vh0N}BD1jJ z-Vzqi{$B6uDt>}}e(U(=$5@rnq>eP~2Ue^118hK$ap`~9|D-f5?_w>*_C(&oHccNY zOu>fquN%9DT{PX%OAek0ocsMFWEb=oy%0( ze6fD%AtOH62EFT}Uf9txy-pA8M_yE}8}{)r|H!r2C5yZ)R$<3p4-`0H_nKwcF2$zo zH@a_&%{Bf)Q`jMUo9T10S%-c*Sz+nQO)gxl`X{x8X4p+uQK}}`EAu85v9V)I_W0;x z#fNK})&E{g7cWu5&U1YcBZHO7k$5JCwbRtk88g89t-I+y53A(O-QS0eweRBm!ftEr zeAt40+;dd30c(_{=vjsJP^gM5#nxMy9WBCME?KoXA3Nv$Ii1(ocDudDe4du&RDrYND{VPZl^H`1`&=PvRczsCo1gKkUVV z$thmgT%~$XSM2M+r`;>D3R|)q7GX^{b_UPG9?Mx4Xo+3@tYr2iY>@8m_j-Td_tG{} z!FB{>E|$c0#!s9+qAx(*PD`45ut!!7tZ%{In=_hNjcvJMp7%-(oiZ6~@u~l!Ha6<;oafTmA8e0Ez8>y__NRh&?3c(mgDPxDjdkZcY-U@_hNsxV zlb`qA#CosR;l^Umtp9!bAT~*E<4J!kr!U0F729*v&dm->3!a&Bv9siJ&gxiIi!mf=_M1?)nvvd#`I0kXPOySV^s{OP0CHSD@sSyKD4@}I9ybHv^)-+jyk zdp~qcc2rYGs5?OLW_rV+V_lU*M|?P-}5TO#$|5-d@GC*b}}D=B`-2bTF5VEw4VJ z+@^+e>tBE3{vH($G!I5XM%V%xMhZ z{1}uMpr`d)pCn>SUrkG%jeW<}xLhuW>n>UKasakAq9a}wtKnjN=%y_0-)jb|=3wog z4vu`15uoJkxZsW0t)aKFeoG5b!KCEJp;%FeSL?*F`9}t>#YhRzzyhf-Rjl8|y$uPH z0wmTN^hOh_A0wlXAR$0bSKG3cu}k(J&OIfL=bEp$tO$0}Vz<+K#RTYgkM`^qQCttL zmqwhiuUZg%akl4$RJ{nmBtNF(C(nv5U?tG<(M#+y&H{AI{BlN~$=lyOP+4aBKdH*+!zRX*0 zd*>I8b`DvTCUwy$f@?e)-ASW>?`!S{b>Q#kxxQK7PNS@w)_U{WXcTGYb4#t2MlXe) z&uMR_QTjE%?598RdUxt8g*V~njVe;68fm1mxiE#-KqIZhoNEv2X;i!ZS*u$ejSi~u zU4*~m>wM!+$u%^p^yhG_tMNMBb!L93#Lu}k^t*EfjXYI<4}AMZBkpY%7soOhu?$PA za!YCS(<*iK)GstrQeJ!YOfkNnL1=XQCmN+ZuJ4=k5no^QyL;pV-p54419^osqDl;G z`EO}7Q}WzrlL8urH9zXI&%>V+d$-*uherPiX|aM3jWnIROAfrIQRn1S%R^q$C`ZT8 z!#|5gB~L4houAQYRQ$ECMJA07{CP7doI#@%fluZXrqf8Wgc2NnNF%?P?_=Ei_6~?AXXZfg7^3Q-2FeV(TKia`(yDHTqn0u3$I+FQRB9A zZOsWZ8gk0w8N_3k?#Qr@rIGJ~sG@CW@Vb+n2lqwOXw|=WTK1lxk@sQbu;nOT@BW1~ zwnu0b=RSA7Y6OiQI!Um;hS8`;+Qs}(FpUO9*h>2Q@b@NM+?g6cqs%|1-G)19wDg6~ z^pk!xdVF8Ju5K${e{*FF%NxHg_p)h)Cynat&+PwqJ&o$cEn9c4!}aUG^WCP^c;5?$ zAKEz5Xx~*=uK02q-H7X|f3}!LchpLjIxnQrnNQVpg$<21mYs>Pn?oamd&-`dXVU0q zzc#DNf<|rKtqVt|&?x@V^yShfG&-$g{7k^c`*e1(&^KMY{w;yYu^MJ6HY()1Qx$QS^c4NBZx(U-^6S(V@hru0F1OH&kSQN!P8^%H%z%&PclO?BQ^o73)c16f=A&xQ zy0u5fc*sV-$NxY-4-J%VIAqh#LxCl$w-&X1(PhwJ-LbJy|@Y z|Jn1wkq10v@on;xZ^=B=Sh8?ZOB@er-Hmygag>L|BUQ!b2lG&>*N?{$emo@7TkmpU z0}uJ6KZ)e7#P?m1n0py`=;U&4+zT!b#h3fE|+&GM~qJQ%{zYL?NjinnuzZph79&-PzOB+TB=WEN3T^UC1@2Ms3r-o6j z;k)dT{rK~>UaMNR45PJ&xhj`d4I>K^2Z;`wVU(1ZFu82>a;+T5ibx$oi=VBX>JvMJj{owQSQavb8Z(S% z7kUk$wpYu#sbxdR=xFOlC#9NI&*$}{oHYL%VORRm1k>K>$@}`z^@C4}_$&HRxRr} z;PBWUG|;?ace8sB+Fp?%Z)wznxUtGtJUaiN;@Zq*TOR#EtgR&`uQ2n+qr9nb|WD_=fQsdZ&alw(_p>-H_|_~IJ3I<7fN=z`{}*!FXXvs${b#O z7jj>V^4wN+p^S>+_WGPoM7tF>1eg&~0M1G*R<(9V7W;LLH4mnqG zOX|>)8E!UrOuwVbS33jZLTZrm!+ggNxmBo?Z&RW#Qi+DwejoLkU5+G>$jH_$Whm=j z>hQ?%QY6?FKB+&Y1c@fRInDL?gzU}FX$1tlM~l}?b6wzDfL>oZeQ~x!4vHD$OO?)i zfwZ)iS`^)Sgf36};Wk`)9SM;?TbZS!k0awLC)vN=qIiW*ZH{u%sSnKSnN-q>n%DKs zFUad}D6fu#S9QyB$@>M=^r^}oOJ^2R^~~o!P>Xln)Z2IA6IIFFr-X`jy}aW~=obo@ z|0|``L%lz5H7g_k|BX7BG}U~xww&Cjl9I|EJbnIj74@3AUp2M69{rkZQ%e~$_x(=M zLMI$@e%F!vH&B*Jspa>I8_4(kpoXrW79TQcq@0-VYozX323ReKXrjcJ@BK;b%3JGm zy7DLa{$^_8>bVyxEn3JrTBwen25&hhS}9xRx!S0!dqoF#{AifG}SnU@#*raGB*|EBDghBmic>L%;| zLy7%LlfD@Ihdjq0YU9k5l7lxr=jx?&-ySWM3hyP)*Gq-V z2m4N&+((|XkE&ejC$grzkCJDex1XAOZNsII!~N7Y=DGW+_|}z{$>sx8E_1#E)bO+g z9}21m$UY2GtMO49~_G6HG^2V62IDLrv!R*TrB`>8FYgIo)&1LpynA+_mGkN`~ zVX{xdRMEweoI|sCWWRWnd1JThohBaT&g>hXIzG9v=i?bZmC2khpXvxGJJCCrCi_TJ zwlCkZHQQ*ip91R6gsiOj=LJ+5v#$b5c=115r#XR31!u{g<<@hgrQR5T=Fl0FpQs< z2#C!%=`Z9g0xag-M8Im@k=ZJ#BH+N(rwDk5g^3!EiNN?dPJsC2y(P2POn~vaodCJ{ zJCf}0O@Q%podA7pTeo=#i-I(BzM{aA+caahizv)u&RG05EQXg>M8gro$dGj$>XMXNZT+8z=xUM~_byieC`>JteVuNw)N zcv)?jl9mKu>PHgRBKiDPo|3SgsUt}^(j;+q%Tq}h?>k9&A|bqfzpNCD*Oe4BPT`xJ z@{$4p(~nZ1;@H=B<+&7?GIb^eu%`>9%S*#}y-CCF#O70Z-qJ8$chYe6Rh>)u3u(w= z>Q5Sq8!9;MiZal`)S(QFh<@p&ePm#~uVuj8^utT}*D^3(monh)kavu&A`9cWBn#2I zUpdeCR~BwCbt(&4@tFpTkt~!j^(qS;Sr0p1)#PBjZskC;?#?rRKRFn$UpZJRa{b7` z968v?)Uh1wGrqJkRzn`5ncR?vJIms9Zf%!`@w%3WvK`T_&+_D9JWu37EG{bJgO&p5 zGIg#1me0Zue&4A8OPP9C0PmV0`#%K=FkbfxPJQvNGu}W#D(5$_kyYO!`b2R1O?Bh}xzM`b@tmgQ-GP zR7{*Q%wYOX85UeW)R~a2O!`k5)=dw0y56o#`cN5m7lb_fPgMoNm|RtXV=h5@Pbd|L zWBO7BlG_81y!KTg{iy=aLU*;ljaMOksshDw+gFynQGseEhgG2E+P|4qohs17^sOq4 za(#4~)Ky9Us)Aa+*P*U|R7oGJ!j!ceTL=7A!J6r3Rj_SycNj}hg_TTStAcyTx(vxY zRnp(85GcDwOZm4d>2ozWdDZElj+PoEFgdRVDHbc5jcnE6A=CG2kd?d4-h8JT>3=mS zUA_3xtV?Pn2h^agWueAD1!^$BJZaB<;US(b&?|* za4Tri;%Ge$k|!GQT595h_$3B)*^YO1?7$O z9&raPl3QBvYtL}C!hS81Us@n3F#uXOv`CI=f&Rsw+lHUDNSa1wvm`{DVS1X;f}ac9g>$i;3!&9mNcS6a#I(4F67L)JxQ13r!E|tl%1UBtV?oK7ZP5) z5YGzNC3&g~50^jl&rQ`OxvC5Kbs42aWx6C^b)jZg`s{CGx+G`yU{K`2)w;=gByaUV zHa=CfZH*quT|MBK-1hB>&?EV)2i93BCH#AOB!~52wf!|K(F#41$9mxR{c@6=h(5_> zeK@o;QAA^^KFMc&NED9W##*OOa#|lA$DR9ZdPJY(wLZL`7&Bw~Lw%Cl`cVJuDW0ASRg8N;j2io&lWPY&V`OHZcJE~b^j<6sv+_dc3rgM@T?i6m zlexl%T1C!Mfi|1W7dEuIvIP+qY%*uq(3@hACbo!8<_#OhI`zCxy0gjLVS~&}T{VT> zY%+h?pdPMW7!jZ1ZZITs%@7i=O9U?tG$ixQ5N>pcnSDHNNama&q)ivCcf4vy z=A9wD3KNO_mSITdo*}#|5VCW7Z%F2!A$*k|8>*`_By-RZ>Rd+t+uUzR=Aj|9C(|3+ zq&Q?Qa$u;Pr?}mKL*^p~M5Yht^~~UqImrRJu)%%%mT}0u?rk z9qt-GIAqRpz%Rc|K539c=B*I~$+hONWQ{<;%v~dhT-&_o7Tbu-Un4kkt;uBaEF&_9 zjo@0-$XqspjF38b9%Mx3vk_$H)%MLhZA9j@5qyxXxt)_@ zMCP>-d|OlHy6CwPncGItc(p>d=#vqd-$wAe^&48*WJKn;5%4U^0?LMs$UL72{{sdr B@3;T} literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/sigmoid_derivative.npy b/test/specialfunctions/data/sigmoid_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..08a40a16ab7c3a1b0688935420173e2647f64d96 GIT binary patch literal 8136 zcmbW6_aoNt--d}MilSkJWR*fjS)E7{NhK|nWMm}~GE-)etVmQSBO|MJlk`u22P_doFXO&PcMeO<5Dc^*gfb`!%L`&n7sSiB@Htz0aeB~?Wv)vY&3%7{o> zpKw0qZ0=}w!r9X5|NdRq+`+}_@4vg)m^)eh{ai|3PEKA%T1Lc2*Hw?F8_t7Ca&%MH<9VYxw)~%=;-(c3vTQknVc9e=0?oiaX#DmJa9TSxGKS#2mgKKP`_NwgVwj3 ze2)k&gunQd$t=%>7+EQ~FQR85ULIFAn4`7`{8P)MPG4Vyx(7M}AJ}+NX;TzvW6z6n z`+2{oyLh3P61MZ6CLfATjRWKF@Igkxtoz>b#b6w)uHuVajAw~8qjfWv;Q7YohZx>V z;4vmEt2MC%FB6r!B%GIGS$y30urEvT$4mNx-^pdj@(L`SKfVm!*UWzDc=5xd_3d1F zb^)kOi4XQ(5`co2SI^Pqg5bK!=3bvGh=kT<*OurAA=3W2o6U0}l&cr|G`cQ_#D`Cw zI*WuM9I@m4NTD!TbuB_atrmg9GuE#mXGD<8D;)B;O9UrQxUxN8EsEIrHTo$5qWE?p z`=sJ)QOF)GEW4_<0`)6|uDpt0fdGf$?72Tz;N;7JHt~aEaAJ3#p;0Y{^)5Yu`z2OF zaPHiPf#EALE9#kJ>-Uv7-??D#1}kycuXMiL)hUj(nPcwRIua<~-x1VOEP;F7wSgLI zBoV$x-gSPeB)lSyMC%JnAz-TP(C-8(cpc)7QWlU#U`wAY%Qa~@?JM-K5nY9PC(8-% zj8(|yzvOJNZZ*c0hFRZKu7*-?GmFu78I+2gFW7wet&R!~u=^`}G^${`qxt6C zZK{xBwOoF=OBK&HF|KYfQ-j!f&gbc$)bMAJS8vo?9dSneF=E^r*fRXN$}mj>F`v?S z&tN0+YuMK`KHrEP8y0?5@&XcsHa+@0UlX^U%3BSXYvOFt4$0smP25xZeD@Wn7D}}I zyT4j$p`rU)Afr+XZd~R4p8VR73s>Fz$6Xusx3tz+cWeLsy~ReQ8#cirUL|Tq;wHrO z@p5fr)q(IUea<8s9dO^j%sux%9k4ui>4?UzdDaGF`Zj!-v*=<0%H zKAna*x>sP_`S^Co^(xFgH_r%9jee?^Wg0=Pf879|T1Of63m1L5@p>uSNIbmt0_sRQz5X^fIV=vTz?bp7A(OgzrO}Q>Ro6*M9I@1zj%> z-w%`3JKMb%9)Pi3{kOg=2cQxv68S^a6ulf#){k;bp`RJ-ZI19LY63e)_}acvL1Uo}hmeJex1{)@2;U_jw7tkLFll zDx+a%lbr>w+p+B{ZnwaA{rx4Wik2`*xm%E!Xo+ywN$-qVR$y$Zp8n6-3QI1Y_#)9~ z1wBQ-&AB_R|9;OXqnRbE>`<3?7*hFm$nlsPqN{2T|MkPBA`SL8d}&*P9y_xU=?tJ7U(F&$9})p1=y1AwTsGC-8^OVBVCj6BY$O zJmh)R370wa4=0Z}VRxY29uM7<*zqQ}WZsRFkX2R_IsD@!Y&UuB^fqzEka?e~W1%zh zYOjo|FLFWZV$FupQ!Y4lE>$S@sSAQx1LGAop2D_;TW5Bqp2B!#+kvwSTv0qJUc1@L z6;|pNajW0EVq=7vxyF7s>?u6|BCy2`GZG3X7HGO-n^=FxV39k7M^{$!$(=^JkNNhD ztkbB}R8-59_Q0~yI&aBL4;ZwkXv(bdgw*@wPjv;J@H$u~{9Dxv1vc*s2I{>qZq8d4 zXzY!FBcB6?`@OOBX3q^7cOP({@98q+@BsPjOB=#(vG#V26M$W=Hu0H3x{yFH6 z%#44WehwdGk3IF9<&X3WIUH=q{Ber&*Q8~WKfE}$hUBgZz-{BcwvW*Pi2nYBMQkbn zae2OehphwgJ#nxz{l7r;n;B-#(F%f^`IYAv@`JEYq{(K(%3uUJp7a)o4@Qi$g#AYD z5bV?)E6h3{g7fq1SR7}bNBrSe5)uCAF{5IWq!9ZB@Ct~^iUePP7`O73Tb!XNN>zP- zFD4X5XSJW035Vg4qL9{|>@Z~c6sqK?U&Lo+^WQd%i{P#?nEGrP4(HUn487@a2np-y z1YNlVw!*^9{Q?m%mtG$?Y##xIg`St^R7N1|g1`M8o=99!&Yv8#j>Pa0hFobyBm`!z zm5^B+1+}<>Ke|p)C^sFP=5CEb_;OayNa<+IT>9lLBP1H0t@~O#-$!FKFRtU(wiq}c zF1w?jAA>A@SGBe!moew${+-`iDvRr5VXr*?`-oB; z)D=veO4H-eKVHuvC3pq1LsX94i@E|<)i1Jk9P!}FmS1HQ6pzPhzvYgzU4?c~iBj9S ztKi9Oz-jgbbWf~ua1Tj9cHW}To%0hhe#v}pTwEell2-~gueb&)sRc1^dDpN|^3_ zYa~O=c8Af0h-5hA%(iVFNX9#EXB8jS6gVAS*pVKY0{)U9O`{Jf2);d-d|Kxw#Lbq? zmrlBgX%(?#tLd9ie`uzJ%VYc9EwM=m26|y`p%*5)+rQP;xv#`KZ4R12DQ0^1fpe>z^ zi3etbk+-vv$-a9=*XkTZoW5htm6rp}`L}homG8j#RAHH0HfYlU1U89=4{*E^hM3!=GG>f!{TGkmqzd4X%7lNs3yB?aRm9u#we6xAIYy(J#Mq zEFXp}E)QEY3lLJh=Wie`z*xVj!(?v(_I{JOnz^nJ>o4)eIR_TvE2C=lj-EpJHhpB( zS$h|~8lpG12iyhELsNIB=Xc?qpI>`h;U0b$jA#4|yNA6)#w#p_?jb_;16#j#5u$D% z{NZt<2(g(~0g}^2*dc83>w{@ALf7{O*WNEiK%t;`o?rDLOlnqzrhXTYuhj2OODioa(3M3xV0?Bp8dZ6hifdQHGv;zusdqI}o{!q*6<1?N^skAT zU#lTEc$`tR_5s?zoYSkadw{@#=t~0W50I%=bGxzs0m}04xKxSMK)hys^6x`6@D|!q z;g?VY!)>@F z+`7rA4$q&aD&&UOK_JI1E~BjuuS7QosyubN{8uA>9txJ7r}7Yr!gBqrZVxfX{!&yz!9zSfQzOn8eu$V&H&4W@ zYC>R2qHMoa6DC$fFDguGf`Lw7>91!^$d^4sB5QL39~(BKOLu9GNl-Hc_R8Im zscZ&Y|3vG~@n%@J8m5-7Zb1R(@iN?$mWi89H z%Vk@kSvY3C*t`|d@k=c|BU^!1p5wjMt@vusZm2oZiq25KPmzLcIPzzkQJY>HTpEHm>eZ#!HAn{VW*v?EZia?0~iJB~$O zJT!i$9VOX`>z5|CqxW>V^2_RWU^){UU$>(;;GDD4)ZeeyS-hrV2?HFxs$3oN4EU=3 z7_-r5z>Yt2hQB!j#}@raTjtJygi~9TR44;2NlR+8t}(Fm=UKlScNo|o!e-7=$v{(= z@z$0W2K+B=be!&Hz)RwZNz5AtMr{I~Ui`>Fx#M@ycVi4xE*i~G{?0(g$^L_-Qw$ir z@ZPre4+AX!K02_Sex6SMpZWF7?`Qrz5Bl?&zpt78{(bcI@X^=De7(%q&-{Csf1fw~ zdzpVf^LZ4}=c7TNS08?75A!@R&lmH&Wz+M=Jde!t$vm$NdVZPbxqzN;eR|$U>G=<)_knpo zp3wWkO79QzJ~8i?HN9^)=>22fN9O&!M(?Wyy}y^~eZEcaH}k$9p!Z*l&I9IrcunU8 zKb;?s={#Z17v{Vfr1NKl&Lif0V$Q4Wbbg(t^X%{T{5Ri9=)Aj4=ieDR4_oPcl&AA@ zC!L=Obe_7?`O2KP=5+q{(Rs|A&%tzF-=Xu{ku7jdgy*&?u%`7 ze{7@sq>1jASh{cK)BUrY?xVAGKb@ufDxB`GU38yar2Fjx-FM-1|LvyxaGLH%H@YvG z`|~y3r=E1bj?sOqK=-dP-N(apKl{;ry_N3oQo7G))BQe8_q`(B|37ISFrxY31I-JM zXnq)@d4icQv}oQ4qxplGN2X{#DWrLYi{_V=G|zC+eDj{>9YdOb&d@xhLGw`u%}a7L zKef|5Wk&N=4$WITX#P^AdF(4OYQuQV^( z()=hy^P~{Xmnt-Gy3zcZM)PPX&8NXMuWq3E^^XL}vpO{2>d?G9JA&lje42+VX+9RA zdHEpC&qFj%k7tp5J)}bNHb2eZZ)qM^r1?CC=5?_llHXHlo)4q>{tM0fWiS z(0=%W_C;6PA6;pmG^YJhf%eTSw0{=TJ}N=`X$kGCp0vOA(LUQw`)x7pyGpeGR?$8z z+e`ZKLE4vlXn(#HLi+R`+OIXtN#9mFO8WPDThhn>j{AT8dHkWU2b`gNAcOLP3d#?TP@eFB@&zVu(4_pKi}DB$$|u?=uNb5JLhdNx85)#t zdt49agT zD9j{thVo3PhN6PCOR}g+TLV2D@7U6qmI0^4#4HoXv3CXtpIxw>@LJA3!f(?}2+xg| zCVbb@obcYOLxlhKtS3BJkMiLO%8P@l2|pG-M0oNLH{r`p1%x-7@)Q2NWSH>i+!=&V zN3J5gI=zSR>+~AJv#&}KzHR!N@b1P^!oTPA5+2^sOZd1~0^#KqZwWtdPa{0tf%0{K z%G+-~Bm8}aCgJgeiiFRrmJnV)@tE-Y*MfxS=P44tFIY=>zvLw0|5hJ~4+zvHexUsw z@dXJNi9e82BR)ZojrfHy>Kn>AiGTRYL3~6=1o0Dl8^WooowP``1**S*(ghV zjeisIHv>P2&v}sKf7UjmlIhw)?)KgK0U zd|5^h@n@~E#HX#Bob%7GZE_;M?bA=UfBxaCe_os-KJ=$F@uQL{#Fr{*5r2A7n)uYO+QhHk z%_F`w?hNs-2?oT+&Z;DScFq9twStFX){y0{M_+<9e#4lIG65o7ja^#e3_-x@| j;- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/sigmoid_output.npy b/test/specialfunctions/data/sigmoid_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..57061d73607abbb40a1f9a360a8376f37e552533 GIT binary patch literal 8136 zcmbW)_di$v|3C1Sv?N`SIH@bj|t(Yl% z6S(Zj2(?VQDXVSU(Cs+4!^3177*+oC$QEvc?j7c~H(Qx-hJA8y+?)xA$L8rrE11Bk zrR$Tkiy0v&+TUDuXNJnIBH^hXW>kiZRhi1NfTi%AS$7-@%J=H3o>|8VK6$D4&DV%q zUVB@2u)^D}T5-Q38{RElVPec;L%hDJun0FhVn6q?Jq}|>uKRX(we=hr(OTBf^W?xR z+t$kFpBxyd7*#B_=LG%8$=e6MaYFn|s1>jMc8rG19VuAYj;H(Tw13>%fxO&34<^>_ z#K|-By>X#C5%Blr!voyA5SHdA|0sJGSiFwb%_(zX>ozr`6MbCJRV%xl<-8mD7o&b0 zW#tBENms z_v(f4LFV#*R;kN;7%FN%&1Jq1hPvIuRqgx0b^D=blFEK092L!IDBO?Y>Sto74)f!l z;$6=D$^4ksdK+)ca{xYaGN*)|9l*yE)xCDR4kE9ANc&{mL8OaaUR%z42%IesZVaUz z!VdNto3bMUsHn|gHK`Cl!EkZi;8{WJYR!;3J|c+rgTM8jxCmin!*z!rjE5num$2O= z<1hl~pNcN12_wh3r{VR%~Ib()5m&K6U8de=sEQSuR!HqX|9)S{_My;gR5!9WbG5s4k0>61m zx=BqOXLMdvb?pad-1 zn5@?8NJ6N@k}JJY5?o2&6w1`3P*l6dLjAoIwwTtLn4XYEEMt@gbGjE9GNUM7~*M?0E}C z@NVC{O0`lE^E>U>Hgg?E&5cx^+X-c09YE$tq!@fhTxfyb@|Auivm; zrwoq9VXl#D$~a{evfs8t8A;sz8rv0BK;Qg0S^&l7mxUR=m$e-h#?>-Ai}p2XeT-&J(&)DU{f;(5ytHSGM` z-2Tn|6ppqz-K|}F8p-~huMI*@V@#jb=N*?iII^zzY{^nbvaN-OqLK!JmT9poW%oPt|$S4b4cO)MCE#&gOOu`*-YCx=zm>E>OFKG zJn=6A@A;iaT}S`6bpz+|>-{d*b21mOEs~zcE&2j3Q6{l>mM&oLl}$MTmo!nJIe5{t zToWRjTMAtHw7_twSJ?M~7FYr|X(s*D!lGpO^wd>tIKJrkY0#pL-+%nFHy_u5xlUft z-#i^O{iYW_e&{0XtXuDnBwj?-Z`IP7ow`Uj%oMba)#BG0Y`U_o#k%5Xvd(mYa9J5JVJgJZGYUWvgdi7!2yk^v0;Nznv0SVdTa|~ETW81c~0sx{hSeS zh?CV=*%%l7B6oN^GseHE?yts6#;~FPSi4jEDxOTO`JIz{6{_KBJ4IPd@TcEmBHq~q z^Bot@Z2W8jM(Nq5}_Vsbw^16SEXg|Xju z>9H*aRCmi+9lnm%_V&QZy6dQ0$sY``yn%b`lph_Sy9pyXHI-KvZ^Ggxt7=ciO(>SI z|I1r#2P?_uwQ72H2znmF9G7DUFK-vNG6s8O)(`V$S=&QG@`aaYojuC~f)!D~j)){ML0xRUMx*$sN$phzJ7xW0Psbkl6g}G3|ql2xkIOapCl$>`% zSLSZ#?lw0Vjqxu9>$*dCfi^HX=#GaWCZ0d7ZozGLszuGuTL7oLqfwv-UJi?tb#r>6 z>V#IJ>}yZVJ#ly!rRs%#>j<_#!(K>PEmiB{eH-oyEr%GGyiscQsMgKI8yh2KD}xHW zA#LKRuEOAh-7}9W^GtoP)#zK|P?-_Ru!dLMOXIg6(gttH#Txs=X-CHOZ*~5-vL=`3qIdw7Q)Yv^5(1Db#Ler; z7zkN?1@{Zyfmn`IGP^n-h;srnqXm|CApY~|;VVOTV4`;<)9BJ&n7I%BsqegtRn^N2 zsoFuvefs6a`mP|@>WrQGdg&fgz-Vbdd=IK0cjY{>zK{53y&Voe@58Vzs9|%!10=6& z5w~Lx#_)gfKeN+=A;}uQ*sb^wds5>sFMfUqS*Ck|JKRE$W0m?abxSDp=k3;NUkQbJ zSGYvXn@~t3eI2;HJ`8zVhC3Jx!f+kQ9O`ew@VIKG`Z!}as;iWC|FRB8?7O6g?jOS8 z@_Hii6aORJS*yBf`R*f38Ft5Sn0Fgs4E9CotUo6A6z48>SgE{w3U?Qt z2n^IUY%c zV?F(n3Aj{Yv`inAfS|~9x8lzUkot5=>W^?D#uYaUwg)6aQgN4=T7M!W1SA+tB%WhF zs?g#<*mJN}*R~|gJ;zaQMSyESP z3RJX0<-|{>;=6=IyVWc?;zA?@6iQ@`9^gNIvI zm40g$+Amd$=OtxryEbb#U%Dy z>D)SbXfhuu5^hgAGz(Dqhq69VSpbv!wpsxqg|N3=XCQh6rBR)M!=Ll0{V zDlo=>aK$vG0yjE#8tnc~;%pY%=jAJ5DP0~mcCQk?kq37ybyq?en!f}ORG~Pa*(T7n z3b_`RaqH`=;JW7GtE^qs5D|Wu>UXmm&u!)(`oF8j5&5TQ@^;oBeD9hXR)-o~v{oug z`B($435~Jq``*E4hJEIm_dA5deAG?odxyo|RVfTIwRp@sBKbDD7JDxl>O?QqB75hh zqlJ3!5fj=GwhT&V>a(;v|HJh6G>g*u!_-yG<;_24%T4$2Ox#{->jbew{7;+-D;dU#$PO+BQRJS6wMepfT`Zfe^SbCtV-iG5zBV~zU zZFn-UTh*zH#P7W4Z{cdkmne?2xu)%SZ+a_yduBUW!cucq|7ZtCSLr1yxef?bn(&+n z=m4YHTK`v#9mp5``qiJa6DEBrWi3}b5#Yok81||YuhW~YDwaC2Rl+w^U%3louKdYY zL%X0Wed}3MUl#^k^>h*rc7sDxFs{Y98)+?uB1Sdccoo~1f0DTeAFl<~-!baJQ}(v` z?O8o&;+D<$z0!l1VY7M%HF^>LSLafEY%h2no0jxudqEwNbkvsnjKQLmzrhbaqcMc9 z{9oT^@Y$blNEYhD$x=V;#+(A0;-tH(~}@$|#Ok)dmYeLox>w9>t*`|;S+ zB9)140O1LlZ_3OE5ZZM!(W`I(zfY8%Ik#yLTN;If59<%&+4XiIw%3DL_5Y3`mYyPry&$;d=hY37=mhTN4mo9VSG|? zi}XoTUO30&%`^h0y_u_uq(|_; zdQ_#ucm$M6^xC$d5fpg0U&wwl0-=J9n%8Q%uZb95ua8&(-3_AuQD=u)Sg&p$itj|m=2mKzQIk;ToOo8AN6bozzt%Dd$M_!y zKFm!*>@Oo%`^G6;*-CF2vTq6o>3)Bg6{g^v9NW*LKZOKm8>%Xb+=mZd55Tt4mpYuhwf-O|2?9h`>CLCfhL#cAlf z(C4)3P2+eSB0cP;VX(bns_X7FcvNKQ#^R?T8X?-!BwQQwoA=`b=*wXOJfG%10PKXjAtM$a;@@*(+rGa zoR%wtW{_eR`c(DV3?9iOOZ#QdAbC@frEB#J*5|*i*xyOw%_sUEPR?Nb&THK_zh?0A za(KtXEwivI{QQG|_blq%?p^W}o&}xG!^{wcSvX22gd3ip#g2eo`E$mzffx)`H{8tl_h0GwEV~o;Zur&6+{qvS;D9W1afiidn2IBHf%DPL*Su6N$DGNaP7VABz|iS7y1nB{{+rqDe`T9 zSoj=D1Psgva&mmSrQaY|~4sKx&9huwb&|4zf zraLePv7Ov2c9V0sd+5NEE8pk9J2|e-{bvrE^|_^abTn)|6MLVVk%omGmo6J|(hya6 z>XXYJ8Xh!Tc$y!gL3lv!FXs}QB zU;3a=L%QseW`-#Z^V_cGSlQ69<(lxzb&fQY{gNp4xJ5(l-u!@8Um7kvJM(b;JsL!9 z)){XPqaosm+?~zQG-P!b-R+N~fyFs~CgddzhEHw8_h!-{)wwbjl|#dMp9em)LK>nw zUkCF4M}wK7u7lD$8W?Cj)yF^5(D>-f_kAri_&EE!&2`eC@l3GkNgoZfvahWKMras5 zOQNyfase`q+t`zYOGHHF*X-ufA?r@+2( zfn8uTg-0>pJ`OTc(Ea&tgBL4>(t8`k)^Dev@~1J~m5ZqTroCei1=Z6HjJx+!cp6<- zeEtxH*OonEc84hhcKp}oAx7blQ`@4W1O?xP)k}IZ6b#r) zps-4QbxFJth0&&0l7%J|f~&s9RGL$0>$xXTYDHmsm_azzhQdtZ&{V(;3UrOff<@lIa~hCX4uq5c%I z{Px5y1yWe;vrkSth{9ZnpltGe3ccNR%`?Fi%AzBW><^{T`atrDb~w4N+EBthf}{f3So9P$6iEJSl~RMpB6**+jmQQQY?iSv9q3$aTE@Os@(BPpm0IchR5JJg`{uR z&SFUvLjJPn|9wf}l%LDysuc1(8Xpop(kLt*@qBSOgF=+1icEJV*=L2VA#SfJxCztO zGiOs6VohRBdP702I3YwRmjde7T4d!>xZb*_{9rx>2DyZy*aFfI8CTB@MHIBMmj!K$ zDcEmnoUSjSkjnJCdruh!tDG@T`*I2^#abUr{v(DsZ2MP1`rFk`FIPpOfzzqkzM8_# zi)D9T)KHL%3pmsGjzUua2j*)jRAq^{Fn^#hB1Oj{SVw`4_xNFjdJ5v%jH72hQb?|1 zIjj4L!e1W8ApHhX_kZ1)mm5jHf7`ZdH&O6x7-v4+OyT~#SD%EQ9~321>86!hA9*eS4XxGzww6CTaQqv*?iFLACW28 z_gB~#3J+V)#OxWRVEtmj{57$mH0rw47=`2exBd4Y@v!%TnesU4yZY0cwL~WomRXev zQpb_QV^zd!3!awplVrXIwSC_b88kl~6PO}(HKdkz-|@hWZ?-Ph5I5oygA57)MVd z{jeJIyGrbsn29SPX20iKVx}mBpEQfMBre74+$|-Fv^Y0!{YpW@XSrZ~8@L4_8C!83pCgA^>c;29gZSQvqbpiA8R>W^^k!xQQ84OBnzY*20 zOZgp{r|>VY@{}X7Ic=&wm+0HKXVZ7$$H&|m5Q23I+>HZmFxqIg;e_|W+WO)(s7DrO`98qVRRwd6*lKR8jOGLv{5eIfHllNf6U@M4~ zAMYs{6Aze2ee@+7`_a-~5Kos+4OjoqmKNtP#JAG}*VgA=B8UZdGmEl_`}Oma>WL5TxYCaiRr!)e{}MkH^c~~;O}@J>g`A>9 z>);AUwg0WP?X-=FMK5H^T#3>vw_k@4EqbE`l8FJs%@U>mlh?Afg{X4=(W(jJ%fGj( ze-p!-6}Yzjp)gjsz{p30K+-cwV#oQ@A5IbZ6^FtMi1mlg{JB9C3~}D*OT2hpwB!-7 z+C_uqB~e{ElqH|YrIBA&OKcBxVCf>Jayzh35I+uI{P2@Fd7<>s`V|WA`o+LTG$(%29Fe+C)YlgfawcZ1+rR8d^l*uL7C_|d)ZZCQ zoK@fF5JBYRFMRowcx$93_c<}|xIkPQag(W=SvGO7!FE#tvEDyFxQuAB>n~MJjQubp zT2Ji1zE`W6n3Jz^v6CpdD5^XH(?zs?mi|je(PK-Ff zJ?cShEy(t`O_UnjzQLDx%kWvCKheg|V=0hWaCOljh-g%=UT~kNpjN!@A@P`mhiWMC zetv^TI5DYuG9`kjB)a}XBvCxqpg)@UcVypW4ADy^VltL!a;|?Mju>=9{9^*~{{ZX} B_HY0I literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/softplus_derivative.npy b/test/specialfunctions/data/softplus_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..74289d2572c0f3f0e60319d4ecafac4ed2b1a153 GIT binary patch literal 8136 zcmbW(_di$v`v>qg4Uw(LNJ*5Dtmr{T6qSr9Lbgcuh>}$*83{>7Nkl1omX$(euk7sc zI;n)_=lTBr3E%U}{dQgF+@8o z$|q#)z*8XEM_G)kuRzNH6Uywdd_YU^S3Tk>xD`}J64%Xa;%y#dl&IXE zHsXTwrLShRjY#8v>DY2*6ZEER8bz5IV1DnEP(V5ZM7eej4k>TOZ8!dO!-371EqWEQ z?8gX=thG~iOibu@o7?JZ!2~9?-#rJ5m|%FFx$V{FEjZ3TIXG^;1!800=te8HfJ4vl zS`IfeB964bI_tv>HSS`usUBujg^yKP9%KPa(OawTL>9c+VW@t54J&pk%D!v9z=|E4 zUijO0u)^QDT6vc;8{RIR+rpU5hGb()F>zjYBz)>+yBEohJRdF}&2=0Y(OcFw^5eh^ z+vdvV9~>B{7*#HF;ROAN#?{^PoRB&mVJG0ih0*Z2{e=r$c(Aih|NG^w$j{}!J+WpR zG>(7kO^n!vpubPSloV$$9gBs47-P`+INEc>K(sSwOvR)Ad%TvvXOd@EzZTd7j+Wa_Lm7L( z#a`o3zF!2DwVA9I6(T4aE~y(lA&PCSnX-pRMA0Vv%jmx6UVNs%==z;;A7qS@xhyjG z;X2&|i3LqDJT`OrS~4SssdFB#=QYH!z9ViiGC>@H9ZD%~U&W#0cv+U^s00EJJO5o<4Im4^7l1Oijtd1{{MEjM&^-kONLuIXYt<07Es6S5I@^5560>4pm zA2p?L+~9FVPL31|cp}c_@f?7wz^&DTHxJNa6h${ZzZof{G+%WbtBdB!Yq{c z``RO@VPDBj`5r-;hx{gS zSY*y|I}X8P^L3k;PoSYLBmZvB3EURoi4_q!iKl`esN5?jVdj=>HPdzyCSMj(d-v#o zH|cT6%|IR0b@Vf>8PLJccidhld z7vdXR3cUpNz;LElEa0{tSVK1GrvA{wqD<8E)OmflKJNHo(xQ*wzXM-xJZu0PgS^nc zc?M|yMJIN6&uLt;Z@n>+avD{?)XQeJ86v|pOVmEj5cw-&eC;`m@X>Kc_;Q>PEXLGA z1$oXOZu_RTrN?Km-|;(V(B88!HBr-UFFp(5F^7yr4P(q}T4(?6HHKC9E|ITJCKw9a zA~ivG4oj`h`(_@UgQU%$4-Cqt&`C65HXShqJ-u0mr-~VR91G)ZV$D!>QuY(woEfl( zlhs_+9H#?gw)#Fa$G@rWFXl_;aG-0b-KKvY_or6<%E>(swWy43608>Z)o(MAZ^oxUX{odCq%ZSq)skIhu>#+KUd5RQTQ*vceH+jQx&F z_Z%^(zFom?-$k^xw}(vDT}0iVg27;0C)`@2dUyBQOE6Q=RC|8<609$=s`q4Gf^sSQ zzr2;su$5_Et!dWcQSKs+0CQb;flg2&d(i}Q6sIJYX7@3d4fmY&XrbtMNQcQ3Di zA7cm(8Y}sn@(;msjEdFyZy`7#GBa9edmU0g9_%|ebR8B(`?Ji>+<>Le;P3j*8(2}j zypXORio6G(AFt~Qg`>gP(JyCiA`OhTF2gsW-oTx6-~JXJHXChq{c#KD>Ovbg2H!^7 znieT%_Am^+N&4~fSr}wklNP&`?|?r&>FnaCJ5bniGi0lGICAXL@1$>vfbloy)%xcm zpxqTE9seo<(y3ntuC9wj-lpLW29ro!L>h-yekATy%~T&|j6zkF%J!f3QAl{3ddKH| z6ue$cqLIA*!+*xe2q>t296zfT_}vkkcq^OF}{0<&{7@i zyLAtLlq?O#m+m20L`7kvZ47>|8VlC%iNP6XZmo}6v1tFet~H=47PflVgMH-V@a9D2 zzlyv#WS)s!Yc6pgXHTrXVfFGp&P=za_DIAdZ1zLi`}}wa#u}_Ur0@VIPIuU;y?ubo zo_uN3Cllb4PJ1mkn1ER+<%vU=9^&rv57*YMNrZX-IsS+7iP+z)9Lgw_1opxQ&+VI& z@T6$0r(Y%+XDZB==|Ync8uQG%4)aCm3nU4_=!+h^LLn?ZwWhOYBQ*kRnAo*Hn zDz+Y#F8-kS1RaU=>MNf<0h9ZAHzDRUe5_TtedKByRw@=#cC<8n-^0Ew*7PYJS3gYc zYJCcIy$A&cUll$B;eG40>WLn>J)wqKzj)|dsA#Q@5Yyr;joL2<5Jm6Nb1Eh&b?IyIS_fo-l)fv3k?ljX6FmJaOc_L$o)PS zM@3ia8SlzNM`n|XbVwee_r<=>p2&lGnDMcuqxm@Nc*#9BJ0EXVdm5P73-GAy`JCr&81`^HVi*rD*45m>0fS2HCD{C!)TT z!BO?E{c87e44epi5b&!UFC?-=*n(ceOF_V>V)GkJ3OMS9K6nGmA+?9{Y!xUdAG%Xx zQh_lc;Xjs7E8x_z&1C!63goldaD7v(gst4`$g!K1WKM*)E_GKz6}mq~cUPe}xY;4Z zs|vX`wu$TNtKhxr^z&@)YKV*7Ne{eKjYkgO?gYK9#(u>I$Md$;Aacj58dldDoVHgf ze%eq2?g{O&i#y-Ke};YLg8y4Y#5Wiw_r1lB-W5+7 z&wGotoaLGk!X?X6<=KptE1EoB6f~pL)!p~N50Wp@V>OU&fse^eE=JE5$aVeEF)eCA zrow@XYCl^r$SS_RLZKBY;YZRNueRdKQPxt2>Q+2xF?#TTt_>mdOScWR+Hg2^q&y|E z4fhAOtGjo#A^7?`p%$KYjKp%B$hB<8JIl*aTv_d4iA>K~`Mn*SU1ewN6gnVUX~B0q zxC2a9tAn05bs%5j%aKY;@R82 zab@@5Bk#e?Uw?Y=By!eBSi2XIe+|wgCG>*d?c z<{kY@eHhtIL!X{F4TJsrJx|5BVf67&nO9T{V^00_ebK35a23t*`EMD)mK|9uiseRd z+kRB7!+ZpkTHNZk&=C~+x1W0XY6SZV*Xv&B83FgD^LtjU{ES>T_I0kiKa=lq>DICn zpYiNfWa=l6&k*Bip7{9iGu%3-#A9ndBT}WUU2Ntv^ehfb#BCWR@oZL;%qW&dUl+ER zj3VP1M?+HJC=RLpu#?FcMcWop{@A8b?3}oMz5LrKytMx+W^j(dp0|lIR2+j>bjqm) z^D%f9X72hIJcftbRnp%w#=t1$W|-DE21NyK2afqM6h?e8R%0E<;PcJ=Vlv|>{}ou= zW;6~-4qZt}-*MFRxMlvgYIbGz!X@%%A2w*O`+%Pv7nz!(_r<^_!=oZ4Nqa)=^o{2 zn0V6Vv>Hw0a2;ZNou^^U)i~94V;X#F@@vPErg1PNOFgw<8umN0x3D)(gWL08lliJv=$y#WsRCen?)Dsc5t!uPaSIlBHGsW25G>eH#FGDW% z&%*Vl>G{T)SwtUq(WjPYVYlKDa~ItlM0=XGg4yOEeeX-bPk}i&Oirzq6`zAmSJ1ek z;vDq9>TF>+ItRmx(b8!KbJ(|aFZ~9KIRx(sk&$zngJi!u5A zkx_Ff6)~}1csPe zKX~%faJ$*Y&w39H`pej~cRvjYYrn95I7q`#8@HsYDh;afbHDFt(QxnCI)*Ae8eE25hCJo{a zYs|SKX}G&z;rhln8nU~KZ}cb9z~YfK6aIt-(+3VxJF;ky>HISmn?u8?Yqzh_ifD-I zd=Vn_h6XETLsylzG%(P5st-5N@bT{FuRB|4xaJY$J=aNt_CwK+_xos=JNUv*WQ2y1 zW8_txpg~dhvcifvvW|h3q3Tx}etom^cUq*uBh_3W;5Q9N1nxfbT}i=zK0nZO9R;@a z3+y5rDcp^pZy02xVD#fH{S{UUWjE<1*Ktu${oVA;i-&@`{;T#5ehNp9H8O7BMIj-s zu0&@Kh3B?ClFs`m1b4h?^Od9^;@-CCCQTt=VdavMJOz`tZ6czFC|s*^nwn9j;Le>C zd|!>iWLLB90Zj_2<2-3^v?wg~YcgEK{!!WQ@O&Lsm1DPdN6N@pn08!q;F zno#6FcV%gk8HLf0&t-}%DBQ045?^Uep{?hpNSPgl>0t)3bO#DEDMM4iP88NQ9Zu4B zA?q%ljO1{qFyKSK@`EQikKM6-u09m3E{&dM@S~7hcC$48DuvRnv3uJBD41T06o?3- zkR8aMuoOb}y|z<9Ka|2;sp!G9TNFNZ*EP?CQFt8}vwv3v+5fi8ef=o%T(zmRPc(%z z`}RZ8F%%-5FO5Bpqwt+`w{b>11&)AC?WqYA;w4Y`#UxS?icq_LC7FV*t^=RRBT|of zdk@J}3gLg*3;sT#pc&}7vFa%WGwt^&z8Mr2_xnBGmq{T;S53Y0*U{g?Ohqc5hQ_i|a( zv4n!_hNkKIQVQu?es%MgQ?Sb!<8*mV;ctmvL+Kj|f5Kgv{#8)e)74I=P(|Sbr+c$Y zHHAy3%Wpicp`e%;e7xx`h2~%me5<8Uoh|Ok{GP(F>{=GlIx=ShhxaMfQ#kOFarAft zsZSNl3BwO$eLlBP<3k5O409+x@Yf%%RKVVtNM!fp_cyS2`&yb7%IQ=pyIQbLQshCiBhK ztUpdHcv7E`*+U_!c-30RUJ4f!N>cWGqVSKYBY2cpTHjk8-$(jpy5y+cPu6c;8S|US z+vTdAK0x8Rqx+MSgQO3hJueoBt2Ft1Vu#4wi+FQL4^t=}u8wIT&I?4**^f}D*(mJ& zkH{Pr@H6r=g*&asg1 zH(wCt7Ogh@CPq6Vm&1+7aiyV^+T)GMJP) z&J#5+$_DQLMqx#MRe*Ne0A3dku7rZ57Q!<#}{SthlupS&!g;#wPM;EVu&;@`m_q7le3ilH1WAHLjno~WR~V-ZT+Ii4(*P1H*tRA?tol_cF+BAOnJ-p##C;ns-B zW)LeHZmO6Qg{@;70*GdTw2a3@t=Ch-)x={hEgqkV`O^azR{f+<@=vygkC@K06mW?6 zpja^TEHU$MA-ssUwgI}!JWd#n#28eEhp zy-Te2)Mj}?)RK!}DIjjwE-0@hwuQK|bP=EOy0T9Y8-`E6|3REORkml{AJV6ONw5+5 zf2q+463dOef+UG4Yt@RBhzWOiQ;!pW?{(8OCf=KQnqfy=RmEBBL6olINe(2MO4G?j z5SePWMEi9Q7ZD) zw;#mb?vtu3|57+Po@KU?_|vLMla0t*SG0ncsBLB+vYV*l-uFp@c*=71s)Iz;{HA3U z;+y@#HCn{|bj#-YMD2Rsb`#>)6rXL@#1Fb7vKNWQ#v*$?h?#44E&CCDJrf@W69qes zw}lbsw062i6FG&7o;)D>jI`uFBIX?yNz5Q_u=KWiNgQZ&+)zlY4=M;NC!Xj2OH~se zzMqk(C-z_5q1Q~zDNsAzNtD_8Pe4@l?GxbyA#4EY87o~{BYs)-jiM?NV>=lUNi{6?_!~+or z*i?y9VtI{6h{8IjU5^p_gRZPQL8N!Te*F}2biVP2J~85Frk)Y;P_1{0F%gYn{iej$ z7KQZ|#P-P^L2IH+K!La&@tm{3UI(J$!iB9)MDvQlC1>KB6?O03hz>T+H$92l#rY5U z5TkeVj`|YY3SatOCCZI)(FYKHOdo~>5gh`3mqLgI=NC;viDva$g|~>%ELn4hcu3k; zJ%V_vpwTyqm|8vgG@7U?vF?2gQ7YG@KaTi!Wanf&@v2(%WCGFhWdA@SG1N(_A({9; DLyh%o literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/softplus_input.npy b/test/specialfunctions/data/softplus_input.npy new file mode 100644 index 0000000000000000000000000000000000000000..f6001e377796ab3f1da30a600262c13f2692f5d0 GIT binary patch literal 8136 zcmbW+&u<)O8Nl&fdu^}RYkPKfW_EUVcIN$2DJjY!1r8Nzc;^IMKq;aUNRdcH6ZL>- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/softplus_output.npy b/test/specialfunctions/data/softplus_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..474ed0142ad6550959c31ef89aa6845b0682cd9c GIT binary patch literal 8136 zcmbVR_dgd7(6_Sp-aaGQB`X!~jIxDfG-PBZUt1v~BT-3d*fW(aBQ%7NGRnwEqG;G; z3s2AUCp`Ddz3zRld%u6X*Xw7YsmKGNY!G=$ zTqcg49R^;omZUqfL(_+^?`>LRIbhcv?Pi`{k#bahVTdz6r(i zukwMb-t;=HFF%-fjcW%i@PqSK%1K`z0caGs>BF=l04#sPX^cVyfirWcQ)`zHs9Ll3 zZ`~4tz%<75@nXWjURg@oC=-Uu&BkNaM?}CUL%P3uSOjibvqh2vMS+8Qq@sjd42**I z#kp3A!NmK&FUn*I5Wgt8=R`OGeuvtz7xWRJ*?HnXjVuw2r&gn5qlxfwV1Z-zC=q^M zk|rgok-*pTx9qnR65Q`UGsw3|0>J219brI@lh_T_RFOT63A=&) zi-J4v#%@^dcWFy-7Ki((o=k`O#KA&-Ig)U64_vTTY%r z;IoR4^bSE1Ue;MOze$$_L0_FHHW4XctFrN&&X9uZ96KGZWNCQ4V5D%nKpN;3J-h;x zWPtnUP9C{I2Fgy~Q+{kJ3pV$N-JIWLA>%|~fX_ua5F8sg|A~DsSQTF}Fe%y#S2%Pw ze;e!rS$VPA+Ub3eka>~er=C2VFG^eyN|%RR%BPw%f8-&*U8(A^!G3r)HloLww;!(Y zwUG>V_CuJ>u?IG%6+rtM%|3(Y3XrB-m@Y1?2)eHnJC;KgVI}g_;G6G?pi`B{m2RK} zey5`;u2m?(fVTDNJEF?KWfCLU8m|oU-gGYeHkBdPE9OzZhYH;CjSG1{ssjA?^BZ}s z4#4F7(mTyv2cWZRRd?J#71CyJ1%|y!scA~oZc+_D@QlZYjP#x4By!tt;qXDM- zB$&>OX+Ygi2XWd6fN(0RBTF0z9Ur~C7@ZI%e~xo?)*^h65FOHG*MtG_`zhhBnsE8b z`I8Zin&9?L%lNpc7RaaKr=4IexG%dFd+W0nyv=ZyW>eP&zmfU!=qznumv=GP-lYR9 zy)4HgFX#ZS6xfON>p%{5ipCeMBj6V?KG#}y1ll6($|ea%|LONGo8CDJTiYaWT6$f` zHVtIs3)6+lp3u#kYr4>XKQ|-DPY=4(j^+GV)B~+~#f7iF`at@dTI0K_4?J|Q-AXST zfUV^2=UKFd@MmH3$%)&BP6DMyV#<~YG*?u#+l=aQ7o zFPea<&%>5(o@1alv%?oreGI6~pT1R*Glhne&4In)rr=gnN+NwS1w(_;P)Rv6pe@Su zoR2nxNu?Fa$Wb$JjlCE6Tg@E2JKNReQq6&<(C3fUra7!J|8n&`a~#qq*|V=T90zGS z>+U=W3wYLF?b4NC0S|H-DY-W+z<%*n;AQ6%;2c;rz}t5MbbRi3OdYWVg(jma+FDEK zuBGwv+iL}r4Qlr=-wIY37|sR~tU*Q75XIB2ff)a1&QjP09=7N@31ry7MZNt)mq;h! zfMjBO^pleiiK#mVil^WOmFngG`ctqAFWf(P>@?h9==>cvb{fjxEu5?kI0K@u7e$s? zZDDNgPlZ;2E$pDxoVWgT{_9D6Xkrgr1MEqf@s^IgF9qyrq#(3h62b%4<0a}M7H9HIPKee#VEM<^uH z^(&1z0&9hz<#*k4kP{;sn_G4c^o`;(=OIH|bEi1S16U2t>sGK_fa?T%>5|+F zaHaO+V6mDfq~#6vHVk;eTGG|u>w*74B}qw0k;MxFW|{9X*?9qFkZw{c!Q!jb0xZpXe2KfL~X%NO3}vQ{!V`N0I^k|1{a!JFs1-zOgQhpnFVdrXh~;ZyMat2Mjk@=^3N~1J2Ek4xQ6p(32qYgJ$+-~`4As^7qo!;jz#yHZFr66!9%kS9JryrQ zM6l7BiMGpNN9^RXcMJtpd)~D9olwvjUChbN41=n>-#LyD!y&YvM){;)IEa-xUK3~w z2Ne#!tMOtHV9k^8p(Zc_ECuZq3qD3boiE=LSGg;oUqoZQ6?X-$y9*|HPhJ5H>Nj6I z4I`nT<;}EcX(aS<3#Omvj)LGLW7PG*QBeP%rZagW3a*OUsO6eQgW=Xb1^rjipc_{t zN!S|$&jxR(C}zdLWPsa~8{Ai+$g`OCaQs!sOUP%|rHO^Lskn-lVX=_-ICgL0RxHqp z4^9h(UIUZZffQlNIGB4uB-h2n0mrYBQcAXXxaquIK#>v;Y$w$k-bh>r&yQYXr)sXl zle;PfJ_mOlCc zRO@T_7*i!fD`7{4@k%ljI;YAQ(xgCavtyC)wG<$#(H@IuPX%|)fbY2}soI# z1}2nl26WHTfJ!Lvmxq2jgaztzJRM1g2XFaW#se}ysBT-Ai!&2MlveqT$}++Et;=?& zWEM!CATy8^%KSI>!cQgSw-yvtw^Iybq3L+}N7~yk3Qm zE#h;aN|ydb@b4T*j8>Zn)XxQBhv7-H!dy^sTB+q^%Y)+-$<9rFd9X2NC3|5w58{eH zdI}tU2&I$Cbsx(g!ZZH&O_^f(VAJjAej_0t%%%@M%cFb**Dk&LGUxvYB-3}PUz&ad zmRw zC8<#yu*ddPLZSO}I72P8e{23Zm>h0D`SQ{W=r!4swoLO9vUG*EOK-mfUbpmz zE2LMDYvte5Q}qgHesGSP8P#9{mll!|9~!CD#aDJ+1=FQH{XzpAa48NF#&}U+73yZ31u+ zQEpCZg6dcw&TETJz|8YkXUgmi$Q2xYw(#@~)S2C~&EaVVOTCMSsDqlpWw^jXaI_h8 zt*^bF(QSeMraDq0%Ui%QR)*o5a4WPG{XWPR*9wu$A&wmDt&q;c`)1bdEv)Jt$}S#$ z3(a;%bS=!f7q4Q$nM{qrp z_b7<78;Ux3D=Qqk!Iy1GP@=jUT%~Gu`Wbq_EhjLd&9(=0(gNzXD|wZv?U#NB2GXQqc?sUR82jE1AWs=(J01TZyxAWe9 z5au4meQxg<1lzX{N;MA;L1lN_vTVT+tg>1QB?^89<}!|Ouj`+ow{_OPW9KsjXFbLO zzhP*|Fy~>I8ip6$O9ImlUqIrNq|=GvFL3U4)z(p~5eRix9%}3vfojXAqwmbVg4@1h zKkPn!h3%yPD{ZM!*xq$vCBuCb7KaZJ-V}|(r~akXin&qvO!ScOR2&1zHip2Zz%dZK zSg5S~atu^QBWPTxzX8X@e2|5~H^{u>y@&nwH`w#XKQ3kH8ytLYLRcbyhvs!oD-qA{ zKv{Y14sY#upo`~`AEg-w*9$ozL1yD1a^~)l1J-f63zY@C3{q+}PII`vau-6wclc{{fy18eT7Ze!zox8d8M5V7}X@(eT|`OiXTYX*1>Mot~DpM^}FFAj_?v!F0K6q+kD z2Z|XVUr^qjgU1H`#NF%v@`tkCDEWDCVR)D~aBd!M81k%MPo9VQmoGH3JLh4WrFfE? zW&zX>xH|tlxbSa3ZwT;vEx`USJns4V3lK^zQKA39i3hQ+aQ^J@1`KhG&qd~eyu`m$%zXs5^Mkbkx#>Omo*rYpbTDnvyW5@eZ|~;9c0e2C_OA(2Z3;_&O?`LemTAbEy%%*J{PE_`~jW{|!Gnrc^4e7|i${W6)RM`9a9 zFWLCHUf71Vk-ajC72A;D^w&*ceH&^ndHFiQ4wPYX{QKw~FfW{0b!^`O5&FWNdkhqK zSW?iRTaE(7-zZZSm{K4TP8SDm>$sCckb-g_q0eV!FJjaPUZ8SKsB~6X0yT|h^k5QvewRKK`A2lj&-|}Qjp~kN-*_)#3ssDKfJjpmgjecV1 z=Q0^-u+*qQ)IGc@QG z^n2ZoYZuaJWx$1lyKq=L39p{rh1UhTUzbGhLX0~Q{k?D(z7q>MF5bNhON!(ceb;v3 z;Ws6cU81yjR9v}U)wu zbSR?UZ=EJfhaq58YjKNWSvnjaI7M@v zhaPuH>2q9BqsP-6^Gy+s^yqKRG9MR5k8Pct?&;<9=%67^S35wDC#(h2zwFQ>PgVdG zlLP~vI&q#=(}V%vpYDAS=FfoJ5-LG0X$-iQ_B@{F4FjfcDL(R;VnCZN_DAnI8PTuF z(OKaTBT8otzs|B}#6Qxlw|TEIq9iv%?VU143>z2Y;2&VbBh#Xna(5W@Nk#+6&Y`AXJz4p474bLR-gf9JM!~e4%oa`u4D(_UWhaJoIcvN>EV#kS! z!LC0h?AW7S|Cra29aAZ;s~!$yN4xa?0Q-1$e4J;0>RKi{o@`?ud0N4aT!$7{zO=F< z&8lh!<1jlaORKl5&a}Mt4N{AfiG&z?<$|* zz{7-Ia}PHTtbAhSly{i}`K|R@z9e$s>=*g}`13h%H_LMyi#iTu)GrN7?Bu|NjXy(O z-#G9Q?F}ckRSujvnkrz*#ECXErxy~%I8n+^@@KCiCw8d#FZ1hhBIDpkUfa{0$YAyB zNS+rb#$<+c`t!b-=xn!k>5p+tn?gM;U|P;z6jKK>#X ziY{EN9J$Vgq#GmN3fWxPk)pa3Q^ke%_Ld$PZsWpldY4rdMz}B~cz!W zO_6!31nMdmenEV6Nls&x3W>-hA%4#)IG6TV_-;c(7D| zAu#1B4{jX&6}a8PgTC)hO*wq#LH@IJA6n;laAdfgEJwqOru*f_Z}an_qLd}ux-2j5 zKg^wbR+ATFj@zWSoZ!XuEaG1|cV5i0^L=?Ylo!`aZNF~a=EZjFt+URLc=2eik9ya0 zUL4vp<*)jI7f+<6(dCTu;%Hg(JkvTaz8x&y7r?@Y1gVFOqeMR3OQv~jsKSS}NirkV z27E~E9_2_n%ZE8JVJbI$`EXap-4h#E`LKgox86C854)cV`Sg_W;rEUl4{YW`tv;XP z;z2%qc|uB2XqFG}b-MqJrQ*l6o8RbGdHGQ+Nz1}Xh9CF1YLE5+KZa1|6=+%T<4}&& z({fjSWOAogc&3 zU!xSn zhZ!$_{pAtFrTPbZyrcy&mQghRyM`bVSmJoBjtgSuK9S~kE`r#TDU_@cB8Xuc9k-s| z`gfwbez|bZLqXh=68tNrRuGLA1$5au1#vFEq$Tp3AYOWQ+j(t85D&1D_gr8S!WsK| znr~u4m|pab(n?7P3n>bP+x3Ld>QbA9<{2UMm%06<+FJ<2dHn=sV}x*)XDmNEMF{)4 zjZOGUg|Mk=Kl_a)A*30kou(QPLds43v5V6}NaY$DiQ%A~B*dyC=Q zwqrvP(P9`}E8V=2EQX%gl1b0*;4uDxQdUIhFRx-(22Ou$LclNlIIz=~2i$IA%> zj4yJLsY@nclho?UN-hBxwca-?mJsl;ThD#38Um);?|2k967YyN9GPq<;D&!BL8704 z7t3C3JB|@>dgb%itSJIIBzHG`Stg+Izi*Tv1rZq@GACIv646swDLjdrhz<;!o*zYt z7)G?PW0fLe{uu=eLq#IKUW+w|RVSjor=fPUE)k$&w5qlx%=BFxk)frzSln(PQ(vV>y1MFL~MQ9K4d*cL?Z`>rMpu^FgM+J-He27D~=acPXC*8y`JamOu}a?Y~5vE|L}&N6th7jyvV|`UnZJ_ zF>uMjHGzbZU&Ze}OeUdi5_8+PToRf}NNkIhkT5UoKczD@B>a*4^?XVr38!kj^ZMIK zC__mY;qE6P4v+JjkC9NbrOqN@iiC%((h}R3NtmV;)x|(T#tO>|th$V36bdmniQ*_1J$ z_25EJPiHdfiaw|=^dcjPbbRMm5E+BK2lVztlW~o4_Z`OsGTx?C7|BW|BQa!KY9yD8 z6jjjyLM8wB*PEKGYshF;P{nbtk&Fokt?YW*$+#r*u#~-@j4$}oc8tf!=#{5x8b3wG UxVXUVx65RdymR8uE{fgwKcS^5Pyhe` literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/softsign_derivative.npy b/test/specialfunctions/data/softsign_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..b26ff4bb25805a52b51081577aede6af5112bbad GIT binary patch literal 8136 zcmbW6d05nS*TyNA$jq=P(a6T7gy*#+b3q^Su1qC!!6HMGL=rC~Mkn{Nle>6^rFSs44 ze17kb-doa@?^{~)=(|GY=k?Z!pC0i?MYkE=6YKq9apOYTmo5IV3D7Rm-dDaqKjP2I z9sy{3wP2a1UjY34`t(V#3xLHxlPumEAApSDMXP`LO!@jVotaVY0kEw7yX5=j0dVMk z^XdYBcVycxOUg`}9D#z5mhFe=P_^(`z56 z2CoW)#nTUcAnXR#79^wuqT-E!>r)E?p>IyRVp|yqKWF=f-VK59X|I0%^;>~(xLaB> z<%#n1mruPK`-%=9t_=GA@h}}a!|&b?cGAJ}*~&jh%+jGL-7TxrO9!oW{*IXCLX%5i;1P-YN)X zKD|a%y&VKAul7$Hor2(SE6ehwFM{BGrLNd}aS)8hPRF%w41&jygAaldf}m;sanRIV z%JV?hwUtlS5GV&NuhnJ(Oo^)RP`-L*N{F@0-S0<(Y74 z`T1QT@OhMAcw8L>R&xmSQ4coOb%nt1&2O#N_0vOBF>#dZNIfb%-kLdQx*pm! zJCB8X>)}z7`1n?!9&I_5?rYPP&quyHW9$JvyvKEo9nzqOUBjB73HS7HdpIa7^rcYP z1!w%-eRwFEZeHjrbPk2lwy9sFS18(Atq-OKghD&e>g(UqL*epV@pmqVLSa>8Kl|aO zP*|?Mn{v8S`MIF81(#n7LucUQ=Le1sgVyn{A#q=X!RVUg2j4I_PW?cqi4KFy*yO6Q zg<)uFZ7d5q9fl8^lCJi;8wQ)?`xVuF!eLhseCgo3;V_I{Ufez-91bJr)X!QTj>aC^ z!+%AEqsC@W_fv)8NYZ+}bL~tx+G^5|kG>ZU&Ds9*E3G4-KQ$&~(-`IX>5FIL+#=98 zZ)x4dwaT;O`T+N|2z1)kO?k5-LYcGAIy5a2_%LKW+!2V4roYeRHtSg(~@hGR#t9vAxTs>RM?IYpPqwTu; z%t&~QRsYJ{QBQxqF^`f zmHPN^lxMetudfe|f&t6+73N35*f?w1dksFFiM;_9*AnLs znP5QouU`G?UtbuI|9&J8QuKe7f zuHf=>1{8)({_pK}14a#QaG&x_wDP_V8*!>%H0-_}dDZ{jXhbBhzvVee`TPpo>J{^% zF>2f1JNYZ4VG$Ag`+)FhxUNaCF3pO@x*uj*`R|X0_0g`+*VRYEwr}iwV|z3@->%x< zqgxC-hW5M__j(L2A58X}Jt_u1NeO>UogM?1i*4t97b&lgc^*EsF$SyR509I_S$Vy9 z=Y^rgF>qX1XRrAs2DW$3)y%pP1HES1XZxRt#k{%Sem2)O7U%v+TsU%UEZpW>kDmCs z@;uYWebdrdc>j5SL*LL?I9dOQx|~>)T@RU5TM-M}32Vn&{1ywJ{{KDU`@gYpINvvG z@N03H7vr+h>isy_)t7cSxx_&~rS7zGsq(yXpvE;k4nvJgXCFdVB{y=Tp4IP0n{ zUd-N%rorECJaKX}G*$DQZa>@%?dXHSs|O`RfA8k*#nX~eQF&rO^-syLxq1E#Lv}JW z562HWemWU`AKi~y(xp5tZeH48pMu8Tz1DR9Dg}0Pd~S5uJ5Os) zfz`|Qvvd2VqAjJb_nMDVVN?BdKvSWe^Qd2RRw`Tu`}VweHWh`*b;%h$(onN-Y1GZJ zX)xvw%H6Xh4TgX7}ItC z&duEdty@lSr{A`~a>zJ`h1TgXI=o!0nU)Th-^S*x)TP7C_=e;BL+R*Pd_JJ{VLDdb z7&KpJlYz^9^I}^^XW*%NCVr8D-mxcKQ&wa^=jrbFS5yYdYX9~~%g?~5y%(R`@=FHP zT#|v?lt(f-hh%d8$iz9y9htaHc_R~Xlru6>MfoC=b44cSi7d_$S!kmCkcHut8?rbr zWMLKMge=%nKFH!+kc}qF1KFGdvf2N$agcmJ8|%pLvoV}}KAZhL2ffMHbFhy5JO@7H z<2kS;|IR@U`F0Nb^;S5NPj6*^-U>VN<*n?;TQQA%IG6o57wgD(bJ0kCn~Rg=v$Cp|8Kj&!&L8q(hq*4+{;BE8*%7}D82_=fa#4|bET z?!hY3(^A&aQsk3yEp(!(+gCLJu}_irD@lkV+f zz1xSuq;vZaM*6lN14!5QV>Rj7eq16QJAg3KuLI~!x^)nWSSf!FVk+s>LDZ2x9YPH0 z(jkl_Jt~JA=}#$!?spZ+ zDTg0J9_8(0e2_w_i=(mk!la>}>Wc#Cpu4eru?tif!$hbQ>{ zoj^~zcPEfb_pKI_=$_SLG~KVC(Ly=!XS7n@JBdkjkAA@;x<9|*D&3n?SW9{A6z8tf ze7<$CpnFk=p8d7^L*E1Hsz4BD5tz}7ISHi&tV|V z?>TtV+@8lAn%DDKO>=sI=ko&3WdlyqJT_nj&0!is@6gfw{LXW81uxUQG+`Y1P7{XGd|YK8xytj%8Y&$1*z48)%_(Y(qQg=S@Dhc8sC(x`hThr`vd8GzS}uOI)us$Vvn z-Vb?yhV%O+@1J@PubAFndw##={a5e#Wz+dA<@1vBQ*+Zr)A@SxdCU2$I@MsBk0m@W zGCwj;GG8)pB|LvJkE;Hf=XE~MugtTW@6VX#y`1M??gNdVMV;xsEaCej_et)T+&A@H zPnzzd+)uf$$$Wq1KCAwE!gSx|{tsY15PcB65dApAdLsIw?(t^**~@w)`lRNPDwBS# zVLcOl6TK7tD`h=Y=i#VHFGW8uvYv{*ir%Vu&aB6x&uT8LFzL7G`C8U@(R5AhQn`^!1@8}T3UBQ<~Thb{Y+I(KG2`}?WBKPiJP`V{w{v+%Kk5TK;7Sp;mLVHo$F##o{)Uu%Xwo7 z=MTvvl26pVw21drbDmM>-<)^UeYgl+nx}iRIUh-0QtyR1PpS8#(3H0%e@&x3zrd8w zR4$OuydWPti6fZvo#Z_y$$y*&RerJ4lozj&&*d=(G3QH_o8+1Dr!(i#9L}dIm)T*; zuPVne=UbKgY&Yd!l@posvE*fyHyNSjJgxF6bKW-PZ_eYA&n2%*ewRESLHexZch39j zJvQ$HD!#$A@e>g`?2iHdfuO9pO*bv_HAe0zxBM2%YH8V zx+U-LNxaX?elPpJ?Ek_8gbxTWuws5t#ymmzg75|#<_~SmBZN-~uMmDw#yms4zne|G zL->dA5LFM&yhQkk@RUa8E5ciZzX*?MWIki!HOz0?nCBES-x1y;{HKF?kckg5FA{zv zJZT>DCE-oNpM*y(WIh$gyh`|$@GMn-;}F5TOVwe8dod4F_pvw=FB5(yJWcqTBl9*7 z=5Mad<6N1~H8QUgekVLn_@3~-bS2YNFfoF=@G`JwPc;ful>HOwD{M+%>GVO}Zx zQh28DP2rtGnSWL>4;4OYWL~QJf|;iZUlrb3#{9LAd93Oe26VNkyjJ+F@LYBN3?|;2 z#QfKhd9a50u<&AM=EuU5g)a+l7XB06|Kkv>NH87=oUs&AV6 z9O-wY?~(pT`XK3tq%U&j{-}ccBp>dVq;Hb`$<#-2KP7#YslVbrOZqM8yQKe;K1}*C z>C043XYSLaUz5H~`Zww0q@R<%PWn6P^Hl#f_kGg;NgpWvp!9{(A4;Dn{bC#UjjGR^ z`$*|0rLUC!Qu<8Q|CJt;`%dXUr4N;URQgisPc6Amm3~$FR_R}*kClE_`daC4rO%aq zSNdM*f29wWepvcq>5rvPmVR0KWY^xx8lOFu4sx%B7% E2UJoto&W#< literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/softsign_input.npy b/test/specialfunctions/data/softsign_input.npy new file mode 100644 index 0000000000000000000000000000000000000000..f6001e377796ab3f1da30a600262c13f2692f5d0 GIT binary patch literal 8136 zcmbW+&u<)O8Nl&fdu^}RYkPKfW_EUVcIN$2DJjY!1r8Nzc;^IMKq;aUNRdcH6ZL>- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/softsign_output.npy b/test/specialfunctions/data/softsign_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..0cff8674b4fb3c2eb074e4bb4c1da212934d9df0 GIT binary patch literal 8136 zcmbW62Uk^B*M*IZSc$#2*kcz_6m=%{L?Ongv7iQPte{w8?7e};POzaS*ejx_=(M4+ zfejTaQf@hC^I8as{oD7R^$Wi39>XC5_ms8Qn)8{f{d@U!@B2lr$+@Q09y%;!$b{Oh zs@85j+^2T^ssOso z_5ZsHmKiYdX2&v_{P)7GP@_~PAD@WJv%F*`cN`hjtZ|7% zE0brYoG!VyNG3@@KNR zF(9OEzD$;x>0Rqqo=gtzICA>X+?l+QlxJdgu1u;g3Kv@NS@Buoe3ba4*eiIfXXFRP zuf5OyUhlo)hecDS#=KR`Ntx%_Hb?PIT#`@xYsLRo_sRMpTd_@IR_nwp#d%y>qQ@)6 z`>Q@oy!}FP*$*`a{rp@p{owt?o0*F7{_29iQmh}?p!F3)aZSQ7uMSSdmKFQ&jI%4Y z8j=w1Wm63AI6e2Sbj3<(74;v*atT9xRy@2=v@usV&McND$-Za4G0t@u-9^>qhsD&Bju{->{Q zC>H)_*U+E@#p~BxOR;Oe;6-(=DsFiBapm00id9pSoewT4UMxE#w8=%q^2M_zFT9}G z@Y>_3hS&tp-6#Xu*>2`UIV$eOmh0axqZ5O8> z^jN95qV=nPtFO?{`}UVt+k`33UVk{c*D}TZH7Cp(y+q$Iki zeZN@bJjJ(RV>|7eqxeP9M%`j(DL(2PT;=Qx#kfn;^j7R0x4O!;srvmtec2&liehTO zr8)_d6eq^DzH~K2vErridh6#LG^56uAjOAA$Lg)$>$=Wu_Keooo0h-Rrjh#jcx`?e zIzq9}#n--n3{#95_M+;rA&MLQ0`*ob>sflY*C53J*Kr%5e{N{E(K-DT!{WV5-|M5F zPhit_(Y+Oah<{{+_Eh|?R(zqMJ@oZ_J3i8>+3$4zgS5xU5Doz>#fgU@uN+H8tTt&F=STR`ijoHIqGy>#mR$f z>#g5&we_X))KnaM|7m>W*NUHygwFA;s`#mQh4!~A>H6RPKJRZ8be(NnarSLl#iotx zg)A<$d1EnM|ARIb{@PP<(&(9e{w}P4-=U_{s~6PgpkU1SEgt&oA73^t zn@^wDWfzZ!<(Bi8OaC1Ee;)VIWVaHF4n25hl3A6%YM)~=^6r6>JF`vNro_Y+$TAtX zJtBL=3zL06$6h#>Y4ZJzK2^$_CJSCV_-U-eWWec+{--lcu3S{GaiKJmp?j)K?3rTn zO~Vq^A|9Je?|G)tqX#BO{rWt-{C$(XTr0`sjbp0^gx)gg+0|z^6HR6%#CbixW^!BH z!dK<5nDoePf70%f$pJOLCXe^Gm%0&aGGBJ$ z*pwKP(N`OuREJI4yGCXIcR+v6iL)i&?$w`b`_J?3E|Y)U-tgqVD3fDGznh}Ao2*y6 z_3fvTCLbRUu_tUYx#xI<-X>dhURgSFy?!3im8#5LV=|?NIyq>i$uytkdh73x`XjW+ zGLsda7T))Gk;&G(=KJoNZ!-2wzJ|ebOkNCH|3}lACTG>DQRc-TCcm#Z_3h3nCf(&b z(PXh>6^0icZ_0Tv`BVIgF#)4Y-V8cg^y6@o=YM;BG<=B3bHzIJYxJwhqj^Fy&jgsf zyT*H%Utj&411cW9)5~Pdn6-C$_b~aRZ|0HP{wBM+m&xR5N0y!Hte;Qm0;9g^pr6C? zybo4>tDm2%&9^q$XUNxWy0i@Y+#tino^5MO~<$cyy&U#~TN4~L{saXb7l2U73 zdv5ULe)GhB(_p_!o&H*CH#ls`(p0ue}~k_ z8+6Oy0-t)QmSE5sa;)36%LXe1$CnR|H<<9gIZSe2X z)n;8^Zm@ssNZ-Os4ffpGeQUb~`ufxFLCc{P1K$`k}$kPg>2r zHOAnpDOq18jWF0CDmpx2u))}_ZOUT)wb$- z8?5YJ29t7oty@u6f8W+qx+5C&ny~X~k0J&eei+s5Zx4f!dpsP*7Y5I5DKM+n2PZqY z8=sRe&w5mu{=!LT?L)S;hLibcm8$)Bx|5~Xb~zpO#L1?*E)$^@(QV(xE!JX_Me$p>`fs?cPK3|_b)5(GUXRp@%-O0199>o1PUjN=N>$Dv) z%E|jia<$+gPPQBsptsZNKfRo+|JRm|P5hlq{@6)xCzCGCdvXF!vh~rgfAeuNqu^jY zTAloBRb9QEoc7h_K+mt8bidB>P6l*M|8%#wlPee0A6l)DlUWrWhW5+jWbBHRwsYS* zxaarU2+wjb{`m5rHkuB$bhV8%2ambN-$Mr@E*1%Gb=$!Mp_xV4HClKQ->-pi8}7@OLozAeVjD(ZToonlFs< zb+DtySnb1!{e^eNY&vV#u>56kIP(m}_&v|)`4I^_I17+$OBkeF;czuPycZgLX6Keku9IhHc2tW~PCq+;@&7%F z-|4?Yy!E#8+Y!EP^VYS~EjKFK+5B|5ekVQc44zXcASJh*pSyQyn38Ry>r|X@*yKIf zxb^9nO7E`QIKTShBLmOaxVP@;D-nln>^iF6u6=*oxX!(7oY0{7ot}$r+-#q!w~f`G ze`w%8+QwUV-o`8nu+gu1^3c_OHX^@$NN`&lcaHexUZqAh{!_U1CTqHg5!J7Y_~Ck-YHA<&fuDR z-3PXxn!!Drl62?Fkb2JGlk<0vebGLH`F~E29@jX7og1|K6j3RI?sil#gXbswSiSz6 zbh^h^S~}C7+luW@OlPobOr1>U?YNv01$LyfrmMZIO6SRNaqCe zRZ*WF>1^ZHLFsh6l$z=6pOX~c#WS4?wCm`Wlg3n+y+}!8p-Q{8!>q5`Cqrk=f z$@UbwR9)foDNNSgZONt-zUe=Xk=t`WrCa{*enL-O_4m3zVMuesw?g(~u5p$5`p0z3lNOIT@Oo0f^`u8E z<1Xk&Jmr!Vg&xsezS|yh$;{|UjusDjG|E-657@h(Ydzra?gC9_m!IwTDo;q}r?M^q zd7q(8UrZc5^FBX*cCCB-xW9CZz?t{BA?WS8ptpBf?LyzT1;*c{zpkz#8A(!CNvzUs zW#WmmcNo!bLBUer+~HGq0o~^50mkK=%v)UNo=3M>tmaHzDmQ7qaIG60`|KC3FmAA@ zo)%|461m7V7M5J+EBEJLVb3AEX{3^$Dq0Ot3lFV+2I+5*eh2A)C}QjH6I}JLh335GEcD9nlET>%^S%4fy^V=t@(r_*1UquFUUNz z?mw3e|5@`6GXEg+5HcU(Z_P`{{DjO?$b5y&TdcR{FJvA=<}+koW1BUm#VfhV+ z|A6=rh(Ce&6^MU<_!;Z{opD0^4#fXJ{1C(+p^W91ApQyBry%|c;e}wo+h`)sRO^E-5_)&;Ih4@v7e}(v2SiH}G z_+5zqh4^8JKZf{ah<}FoX^6ju_-%;)hWK$gUk1dlL;O3$&qMq@#P37=KZFNB_yB|# zK==VZSv/a+U=!XF?!0>URCyaK{6aMa=%a(+z+?||?R2oHhq5eP4V@Dm)fcnXBC zKzIv;zd(2lgwH^D4TRr7cn*Z`KzI*?|3G*UgbzV@5riKh%;HH9z69Y-5dH+=Q4l_b z9u}{H#dA#v&w}tRbg_6BW?TFVmn@7H>o+i$6kmB!o{wcqN2iLU^V% zPm@dt?}YGAn;-wIN3gM{`z6#;35dI3`vDj+yS)8$WErj1fc&^l+g798U zwD>QsSv(lRhatQe_bq-5;mHub4B^cX{tV&K5I&7uF>YQB;n(<|#j_!N8^XIG{2M(j z9uDE-5MB=9=h$fRbO>LE@OB7)M@@^z<8F*yn8<4z1`k{j4A&`6ol9xd8 z6P&X06iB`T$y*@#3nY($-?oCNZtv_KOuQ2Bp-$3rI7p-lBYuQRY=|n$zLIPEF_W zc`zg&hUCSN{1}oaL-J)v-VDi~A$c?;pN8bsko+2wXG8LBNZt*}zae=zBp-+5<&gXw F{|9z;QOE!Q literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/tanh_derivative.npy b/test/specialfunctions/data/tanh_derivative.npy new file mode 100644 index 0000000000000000000000000000000000000000..4d0be2e26066b5037586eebdecdef6f230d13620 GIT binary patch literal 8136 zcmbW6`Cm=@7shwjl!S!LgnKJ)u1u-qu9F63$UJ5!DrG8>MmZv7E>h?sB$W__Bsml+ zrP6SbAxc83i^%+4oBfQj!vr=d+3^I z>zXFR6iI=gwS@o-qV(9OfqN!)Jbuza->Z(r@=;O@k0y&(n$L-YseYp>J(fB)13 zL72Tc#q4jUo_xg^k@A**nTxb``S6pWA8%TS>+IIimLe-YZm<$*-+RSak^3GwSpVF< z-`+-~_hr{{A~nve8!s~H`9BjxHkI6(DAMKQcUzH$&V46~oLQUmlRDFUrikmc&jwEw zsrCHLPd<)Io+hqy12#+-IrQ?Z86qE?H=OzNd%3-5iTpfMW46eEt8M0pJg~IGT#3$cUUUdyzHI|8@{rp!x16dmK2uNL(MzUbR@{ z#5z+)kq!$qoJ6j5tNqCozanRG{kG!j5|PD1s*6Y!vou$cR&8!A6`7I$c$vr+UsP6z zbWj_zQsif^1jls7YNu}WO~w7V!$p5D*hLtG!pGFUCLz4@n~)N9CDBYy7M z_n4=+J#TQ(T5fh&APx*pu%CsONMw7OktWs$GuGLM`86M?eArj zL@?#LR+*7ZnS9ITed6}Wx-0vc(N<4#4=}aUa-KzrJgD9BAXBih8*zv!Z)|iw%$)oo zbzZc%ev;7Y2s8Rh#3hmLt8aK575OM|*fHkhhF)JqYCNdPi(xkOGmbqju8Wq0#4=^u zi&mdtDzeI4;+O^x%9q45D=chQB`_x|wgsJJhI)58p2)NxIj8UxGkR}Ht20b@t!6q& zOoh{>J0crqObs~8)F|sb;v6$I%l4hfigx2KB{RJX?1EF63is0a=b73C?MGi=Ce3sH z<07-k%BV{!Q#e$kc}d*v|F~-!vtek^fXhs6j|rCPOncwBO`K`#|Rl>cS z%-I7HZr)-x+}F68&1|}O;93qdB=PK>T&BUh%<9`r)3Fw6`OHww1mio*@I$LS3z&!c zw_8}qd_SwJS`l;Rx?brb)8)>dcbQr?WLV4`Q}*Gz$Tbd63QCx@0k-j_%=fZ`+sc?L z=jnO<%S^MKy!;+>c974q`^=in>YfjnTD{k9DQ6DctQ-H3IemXb$s?w^%HQfwn7vYL z^(vV8|Mn~r*<$w{`BP?m`N@&bm_Y}BG>UAx(bL;Kk)J9^;F6I$m?v?t=F5F*1jWeeB$jz&a3Z!=CxUu#8+RKd!|os_lfe0tvEQ@>W+NLE##_H-jlJI`Gl!H{!D7`pT#ouM5AVt2F2DQTVmH zA`J>BCbr$4x(Px}yzmFVE|E zN}adszl*=ume(PVd^Fng@lB6x7^uN(3U(@H3ZwG}ZS26?bv^o-X!8EA9CtP`E4m)L zn$eNJS30}Zj!q{t?9-5vdkt+l_+fzLZ9F8weHLN zKaNuU#taP(exKTp_c!t1=HH)xHz%~0_3tE1HrM{%;|~%Hn%8b@VrmSd){k{aXo&Av zo2N^HyMxir8+y#;J~OfhkPy8m=Va+X{@$?_Ki>Swzw0v~v%?@>H>_M~_80$N%+7V| z29sdDsL^P#0iRD}XV-Q^NT_LJx>v!}{yL+j?@&IT;ds?i!+3v-9k6@zIU@i5}+Fkq{El`*duezEW~;p6L6SL_3cO=-(?OmGPXSF#QQHVp1s4F1eJd2hrO4O&|N>sXN3z1ox4TZEO#X# z({H(#=TZ^|UfI2J%Q6zGHheu4x10q1bKTYzui)bjYaOWOM)Jy`Rh@O+`T7~23n*Zg zo{6{`u!@9F7d-on@F02l54D=N9{hj%v~*5gP4a(L2EGhhL-O4QE?IV-By5`6+}30* z3F^C&pY-)2x!?G!Wt!e3tSZ(EZ|g(CpHuGmYxqwYrS}Y3>AbH82sbN)^_$vc%kXg_KLDX!m_kE`22g5}nVthhiD;%`pL zS-O#go0q#BAHInco*(_=S_YA9hg*Zr(;$*B-O%4FYcnYd>h<0w1(WPVRd`cu2w#V< z2Ww)tkRoXT>7Ct5vXOOm+Bdh6%ys*-(be1ed+Xl)*I_5g2EA!{&Sn>BHy^L|)hd*t z+l*Uqq&}2n&(5T(9S$S8x!)B38M{f5xv$Z*;~tU+9`6-#Zx6{dZCaEZ4=2UPKQb-- z_L3}gp~L7o5hO$hw2$3CtnYP@DtdV6+s7Uv#gnLGVNVZ}K@T_GsPJfNm>Zj4`S%f$kG>i8wEhUm z>RaqDJ#mzxx1MU#Y{@ZlZ(|d3W>5@;HkAqezs8Vou)6rf-Qy&4*jK*wY%GPYT;<eB9p!v_dnwFEt6&&H=d15 z&mxVlI@d=A-XNFY7pGU+-lR2d`g%@%Z;{dMU18>5ZqWq$?G-~FX48b?&t9{#a!5TX zStw7=C3W9X^eo{vHNU>FIxIeq!o5#Bw@u2Y>1&eH)@IzHLdVyG>q-j9Hg>R9j!z*C zS9kZOwne12z5dS^F-0^^Se<=ac9&`^?UsM6xJ#zHI!Ag06;nXYYZZ&XN=PHy&da^L zgg&2ZjQhO3l=hjtYF0U>j1D9?ZPNW#Mq5W#1ZQ6Umjb_p9Xby7X5fUMN_w|`jifF3rB30t31PMwBbSkr6!L%KKO%ijG<9#P7Y;`Cyx$Mjfdz1xWa zPsruchpNU_6(kSed-9Th1x?xWN;C8PuFDUxVfLQH}7bGkQ9;{|uL#oSm?T$KEL*eGf z+`F=>B&T$GMI}~UPtDl;ikek^$eYsmicYDoh;(##O^>55MA+oM zrrx)^C8iIqrQQ3xI{Jpy(z|8NC(7z;$??a8N%_`qXvegFTRuDZhW!8gr>9xn8(JHb zd-#lT9W4smF~4zr9aVX!8$MFh(S=Qc9wWZiQJV^rwuM9L$+dgJZyO!zNzXlhx8AmT z()zqXe)x1f<(5pdxqZ8ye09oZgg>jN^p#D0Yu?sV+^WiXm%i3>Ul9Kqlxb)!>zgQDZ(M)e58R*k z^MBnh+&|n;+}~K~e&hb*dBF3rMtWZG{NQ=Q^M&UvMtc76JmUGp^EzL8e(^lx`Ns49 zedw>}AMXR+54)Y zQFHBAJwbgzy+QpklIjua6Y3S}SJRhY^$hh5^$zvVQL2ZikEoZZpQ%zkMSVrRMg2uR zMtw%TM*T)TZ>s)P-%;;T|IrUvKK|?v=ojc8=qKne=r`y;=tt;J=vU}p=x69}?oz)) z|3g1?m--|6CHg1&Df%n=E&4C|G5Rz5HTpODIr=;LJ^DZ90n7)=c|n>VFi&85cU=R3@MnEx;jO7o#|UXfm8Pt2pze5#yRrTJAk&r0*Ha^A)Ki+LFH zG3I5=&zPq%Ut`|J{Ec}W^Eu{q%<2z>~>2>l2> z34N*5o09&79)&)IUWI;D>RCzOLhnNVLJvb9LoY)=EA_Obuc5c0zoEyW&!N|$-=XKB z@1ggh|CN4#1AYhm5BMSQN8p#hKT-NA zlE0$#TO|JlehmB>rC%fYH%dQ8@^|3(!2f|Cr1Xa*zX<*j{3Q5G@SBwW6Z|O2pHli& zl79t1OX+V(ewWh!f*%Hd41O8>Gx%xn*Cf9U{+rT|gFgqq4*nhdJotO?`{4h<4}?Dm zzYzW*{6zSR@EhSj!jFVM3BMBlCHzeIoA5i~f5H!iKMKDT{we%a_^a?+;lILLf(Y@Nts7U`4sXh1@bE?&qBVX%)6xg3wapwF=bwc{0w;-@--=ML;i+5 zPMOakuaojSWu7PHd&;~I`5*E?DIZkkg~$()Cn8@&-iZ7Wc_i{l`MBa(~ z6L~1|QRJn_Pm!l0Uq#-E{1tgD@>%4yT2g+CJQw*c@?PY>$b*p&BQHjNj650nGV*5R h&&Z>ZPgh8JHS%lZ*~qt%cO(Bs9*%q*dHK)${C`?wk4*po literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/tanh_input.npy b/test/specialfunctions/data/tanh_input.npy new file mode 100644 index 0000000000000000000000000000000000000000..f6001e377796ab3f1da30a600262c13f2692f5d0 GIT binary patch literal 8136 zcmbW+&u<)O8Nl&fdu^}RYkPKfW_EUVcIN$2DJjY!1r8Nzc;^IMKq;aUNRdcH6ZL>- zNDx9PNI6i%jY{-D5r<06fm75&59y`vP$HE|Aqt`;r7D$^G$c*a;3WQ)#Bs3S_sc)P zT%xSU^FE*FeV=FFcl_ILJ@d`|-)lC0+IV5tvxkrW;MlIOh+R(|d2-htvFph5$9{6` z(2u_V{IO>b?|lBXLq9xzc>D9?&mB5?c>DjmpV+(ii9L_+5x)@s@6~Cz>#=%m_krKM zy{hY*PoF7X{7l!EPDH1_eo5CCUpu<5b5YlSJ-2h|`p>I7H|l!luBXrFdi$pBJ9T|( z=eE~${rgvTE?sXwZ2LxC@7z_qsOuMY9(O|5Ki;|RsIHH=OV>N2?mnRFZ@UNT`WtRw zUAx=%>H16V()Hu+MqQWgfx32&i*&v7w0#|2dxyrMa3~xKhr*$7C>#oh!l7^|914fR zp>QZ13Wvg>a3~xKhr*$7C>#oh!l7_T91@4bA#upsrTJ}%L*kG)Bo2u~;*dBb4v9nJ zkT@g`i9_O$I3x~I@Ioh!&k zK^}_wB{$&(?gDRtv#7u73ZlSO;3;qv%)=PR7{?gL7{?gL7{?gL7{?gL7{?gL7)OpH z$C2a6apX9196630M~)-Mk>kj59BJ*^{AU9k0~`Y!0~`Y!1L}2vV}N6TV?aF)a13xHI1>6sf+N9^ z&>s>U362CuLj5N=5*!JR1V@Y`#u4L)Yxni`N{l1M5#xw)#5iIcF^(8Vgd@Tc;fQcV zI3gSojtEDDBf=5kh;T$W`Z)SH`Z)SH`Z)SH`Z)SH`Z)SH`Z)SHLL4EE5J!k3#1Y~M zafCQR93hSnM~EZD(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh(ZkWh5#R`L1ULd50geDifFr;W z;0SO8I076&{SUW~S|skp`j@U?7KwAQ{-WFGEE3mZz2x>^3&gQNeJ(KW0`<7SI1BWV zdGa$)z0K1v=E=i6{b3$=mwwR2*{$8MC%U-0wc8f%?&9d;n8PuLV-Ck0jyW82I662w zI662wI662wIA(Fo;+VxTi(?kYERHsgHjXxqHjXxqHjXxq85}b>W^l~ln87iFV;aXa zj%ggzIHqw-a9?04OH;E`>+ef=Zr z&ly|xId@*zvVl8rY}wkKLmpbc&w6NuI~P2(7u?2a)BemI5H_t}znk_0SAP%euU*|e zuzOv-J+Mn|!QZ!jUv=MBZvS-OUU%o9d-iv(;or0VTyoEDxCd_7SKYuHcE1a7!#3Qt zVZCD=$2yL69BVk%aIE22#j%QG)&9jfR&cD~Six}@$6XwEaooXi2ge;8w{hIYaT~`n zj%6InIF{`}HxK`@ue*)vE#kdJoS&2Dn^w4U*-d-WZvcqn2IJqbFS-1EYL{K!t~1Vc z@^sAxZl15%gKmERO+K!YhpYCao5xS=1vh_J@Ls`r*)F?zx@0SVV8ye9W6Ao3`LR9d z4wU~Q-j9g$L;I?W@1M4E^Z0>nxOgrS$NP-`o)vDr{M|0O@&3j*7kK`8Te&=(vnSp2 z-sL&(@cUeUs1MGK|O1&l>M~$M`SIcMb6xpP%!_ujdN}pQ*f;-+#~edV8PeTr>xLz6|*@ zzK%XLem#6-PWpWF{Ev;Vk0r)gGW-2}8RpOU`nzm=U0yLue!dz16XWaqDsfyjNBw#+ ztRLg+?waxSc-^qxi0@P5>-q+9-Y~3Jv)8X**GePP-7K5^UlzHuAhZN|TY^A2&`F;BWY-!;B3-8H^X-8H^%tr*|OR)}wfJggGu zD*0G7sq2et#`noJ?SyY1XC^9N z2NPA-^^wWy%Wi%reYqW&to*)jvhwS7vhwpiS^4^C;b`G#;b`G#;b`G#;h4fPg<}fG z6pkqzQ#ht^OyiiwF^yvy$25)^95Xm(aLnMC!7+oQjiZgDjiZgDjiZgDjbj$aERIaV+8p7$>N{;QC!q`S(OY<=?vom20O* z0(=4E2RH-b2&(h0kb>&6+cyQ3e{UT4vgq{i_lU1Y9(u&tBOkr$S8l)1tNi=oUgg@? z>0agEXZDD{N1l2(LL4EE5J!k3#1Y~MafCQR93hSnM~EZD(Z|up(Z|up(Z|up(Z|up z(Z|up(Z|up5#fk%L^vWG5snB)gd@Tc;fQcVI3gSoju=ObBgPTqh;hU?VjMA!7)OjF z#u4L4a3nYq90`sDM}i~4k>E&hBsdZr362EE0LK8w0LK8wpgQLE8v`5z90ME!90ME! z90MFFj+A;&tEb#KAf@h8>N3TV;z)6%I8ypSO1-A^g_OD-;uzu>;uzu>G9N?gd06>< z#E`lk(m#echB$^eG8`F>3`d3|!;#_0aAY_#92t%bM}{NAF~TvzF~TvzF~TvzF~Tvz zF~TvzF~Tvzk>kj5 z!l7^|914fRp>QZ13Wvg>a3~xKhr*$7C>#oh!l7^|914fRp>QZ13Wvs_acCSGhsL3C oXdD`c#-VX&92$qlp>b#&8i&TAacCSGhsL3CXdD`c#_?G7Uy9B-mH+?% literal 0 HcmV?d00001 diff --git a/test/specialfunctions/data/tanh_output.npy b/test/specialfunctions/data/tanh_output.npy new file mode 100644 index 0000000000000000000000000000000000000000..9f057c66c425171106a30aad74ed930f0b5b090b GIT binary patch literal 8136 zcmbW6`CrZH|Hh4d8Cj+<*6cCax6sTrVU#V~#PG3YHzJdz#lDt(Nhm@njBV`7a$9Gj zL_{T_MWy8&vbOPk-RE3?!M9(YkNYg0bKdX!e!Z^Ob@@BH3>q@(yHX2EEo?b?%G^mF zEqhqEw4Z9%($>1=RCkYg9NazpaFw&~EmeFxiCw$_WS z|G$5gySU#i`S#7IiUyxCu|0xJ{M>Q5i588gnfTw=0VbBc(ayxqV``c>==S?!{r=mz zk4@YfdB((s{kNFdf0CPtuH_s}{GqY6i5_8}i}d@lSKT+U>#-0MCsvqeV#Jz0CPrHQ zVB((V z+WA30zPg&ViQB&&e6QDwUp6suw9lS=y>1v--9(?#OW)~rv`^++&2P8*ndlW9YSim5 zw+aoKE8aPo_)GHUJiQ*(^Fgj=&Xw{e##ZZ^qt}~XjLFutedzN>^LxL|S(-7SL7AG? zcXnrJ?pnO}jLYhjs`)0c_?2eBj!Q2!>yMuHLUZKkipiRbE9_3v z%!q3EOf#m6e}ZPM=AWNxe&^o*iRRnx8y;)cd2l&av&Wo-2bx1irrg);bKyyp=C%vx z?rIizE&fllwpF`3nwEi)H#Iv~>UUk!bNs2Rnr+t?Mrf{{)9R9@RaTGln$xefIitDc zY^hV4zxkXwruoAqha;Nx%bqx-`6MHMzvk4$8o`<+eJTcOc3cs+UDG9D-WJX9MVae0 zzie&otJ$fg{R+)q-RdsZOg((lQ!~-2lZR&Y$1~hC4`oi8s%fxjG(mIe#a*K`ExW`F z(_Eyk4AAuY%e9Z@8pm*Z&7%!(bk=NmcD}7<#eI*PYE~E(Usp4_V?8w=|JkLL=DFyvR+?6GzkR3qz_wjpiLlAs z)$y7=rxsk-+pR6OdbIl!yaSk0E+M;$bu-~850bJ@<;6*L!JYmike zT)#B>Kh4#_t->_(hqPR&IjPshahd_E?%8P$-!!y_X6?@QSw+HGA>LOtD-HXyMe|_g zu!)-UV%+ri@S7}=K z=Qq}DwmB){qwwvD&4V>N`=q}7AY61lY^LVaoyYUv3+ryZIZv~U#rb#n!sFwdXJ}qO zFe2%lu=N>#M@@%vcTc<(mh8S*RnsQD!8D_A<=%_o2I0e-KbOzXKU9eoy0u(t{Xn?GGNbW*VZDP3ETV)* zn~Z!NDNHC88v3trqT4*@+rq6M9=y0A3~zSHMG5!TYkKC2e!O9Eez>rSP1D*Jg{?f? z8k`eWDB4%{v@oNbchm{twSdq$e+%EH-pLOW+DFYCa!{Dqv##Gh;mm+OyLJl?RB~Rr zQ`jZnucyDT(7J2XCSjb-D%*9!miso1T`gSeGIq=|;pp{gEfxwt*7_DPUueC?v(_Bp z$Sx&a{}dh`Yg2c!a8Dhtt7C;9M-6Q_Qn>d;N%tYb&DD=p?I*19a{3MjVaYn1*zUsV zQ!ht!5IXwC4sIn}r}n0u+`9CA`8CCb&E;0GzxDn zQ}5D*S5o_oP7ofBJ=-r@xFo3FWhHF%apci6!s^);)k1~C4xG%`A)IMFv$db_TuRS4 zFJaNt#N_G1*#{$j{X-Zvq~@o-!aFUaKXn$m7p?8tM7Zw4{wo&3mEEm83QOd5wF8I0 z7WVDr`^N*Jd*@f4mxK|?eUI!H-mxow=_|alJIQjkaK`Ep_QQqko#zbbCUiO;-nFjK zrP9FsqGF@kdHD8U&xJwTapbaa@w8!nfx<(P<;yM*?!F%P{ZQe@s6GCzh38I(_>~q$ z{kS42p~$G3r*v3&Lg;7n-es9^eZ9e>h6@(SgA{4Z*g%i z{IuS}Nf;aI6IFi!;jSj1RK36}pb`p1;UoRCZ~;r|#s*KHM|O ztAg;j^^p}0xkeS%{NducIr2ScS=9>8mg}@=@Q15!jH>LHhgY9u3B!9^re}%{nz1q< zD?`q4#Db*M*GA=EC3E4UbXoh4sCY3=>QQ1`)h($;)n-$y<=7NC*9Y?UH+UuWbL+kK zu`lHF9>o9bnJhZUxuJcv=kmJ3Nj?4pN052e0OXpyi!#;C@W%8Bxemb&HLE_u#9qiPTu+_`t8-2d9c z^1t4Zb92kL&B0sp{kQ%4q3sQ+=SkW9L#~PL2rXk$3G_>I<{BN zk3&|?@F1i5b3&uY;2m-<|Ek>Lk8MV^uS&Zm85^bFob;*TxK{f5$)nZ0RvA_C{G&TO zm&)ff{dBswkDS{H??*kGC;h|g*SX!@W#4bF5HoIu-1k;a&7CI6{a9YV;rlU0l{c-h z@7NKd%gP&Ttr{%nC+$h@WGC6L*6S@wewFK4_Qxarx=Wo*dv(vTgS>Bza{FLoR0}(_ znKsbcsFFNKEf`qasK!<3ZTGIK{Qgr>KXof>RQFy*o~m1DP**=zFTKlXP?h$q{^eM@ zLB-vwH*QFxLH)QorOV3u2KAj!=>@o9P`&C!9a(nHpt@Z?)}hy7gL1mxbKufIgUYu1 zrAnvu1{J>U@W?+F8I((QnDLF9LH#fG%#_qI2DNENP$T<+2Ico9*yUXhgF2nNzDBW) zLA9=#-DX5BgWA%kg-bzcgZhy9xYSESp7IX+HGY zXMI=i$W!f3jIBC&NuF{)UKG`CN}g(w`%8SI0ePzVG%t(V?ekPfkKp=ME%TJ;$5~zl zpK{gf#6}#XQ(&&LtMaPVF3((5C*(n4^zXUqO_y^QuD8!s zq1&f<Cs=Ey7cr`soo%5U&cKS+=vfDk|!`?qfwXSoj>hS3~YW~QG=d>Xj z>0hemsH!gCyL-IIRwtg@%o=ziTRk{g>Hb2WY~|KEW!A^vvQ=!UKFv>7%T{TY>UL24 z8&$U1i$?zhzfs<`!)m6vzEOwVSNqvCd!r)5udJM(o~6=e|Jqy~$Wq(iTb?f*m!wJnpCcs%@-a!NU#Hp=0Z3hcjOYJ-h0)z@#6Y98G7LVZ{_B7V4kvI-hnc8tU3 z=c@D5eR)gQB&jkp6_B zY^NrDj8g%78a4bB^H^nBCLT^c^hm||7rp4~8LNhDtaN8W--l}dS!dtyW--cbq8veef=fT&5?+5=M>baZ# zKlpjz=Y!t|=gr>_{J!A#2Y(Ov`%vD>^n1bI4_*)O`knm7q!TJl_=dK0{a!%zrcP5_BRYR_dBrvf&CB$bAJT;CD=c~ehT(iu-}6H7wpGi ze+K(C*uTMk4)%Aj--G=h)B~VC0QCZ>A3!|;>I*zI>kUwUfO-VfCn#suE1-S>^$e(Q zK)nO%AG|T^Ay6NIdI{7|_|2@RKz#-3El_`fdJNQOpk4#@8>r_%eFy42Q2&8?5Y&gD zUIg_cs3$>v3F=Kye}Z}x)Tf|c1@$YaXF+`n>RnL(f_fO#$Dm#Y^)slaL46JCZBT!M zdK}c}pk4>{JE-SDeGlq=Q2>05~6j^8z?Ofb#@6Ux4!lIDdfi2sodB^9ne>fb$GE z-+=QDIRAk25I7%!^Ab2ef%6nNUxD)$IDdij7&xDS^BOq6f%6?|^;}=>LFz z5aLI!Am|T*ej(@|f_@?fnEgf2Zv_2E(2oTD zNzktZ{Y%i#1pQ6W?*#o%&<_RuQP3|1{Zr6S1^rdfZw38V(2oWESZwLK%(2ocGdC;#1{d>^Q2mO7}?+5*VFb@Fp0WdEB^8+wX0P_VfZvgWL zFpmK92{5k!^9wM~0P_tn?*Q`;Fb@Iq5ilZvyitFpmQBDKM`B^D8jV0`n~}?*j8LFb@Os zF)%L!^D{6{1M@X7Zv*o;FpmTCIWVsS^E)ul1M@vF?*sEcFb@RtK`<`_^FuIC1oK5O zZv^v4FpmWDNieSj^Gh(#1oKTW?*#KtFb@UuQ7|tB^HVTS1@l!fZw2#LFpmZESun2! z^II^_1@m1n?*;Q;Fb@XvVK6TS^J6ei2J>YwZwB*cFpmcFX)vz_^J_5A2J>w&?*{X4 PFb@awaWF3j^K<+k>UOlN literal 0 HcmV?d00001 diff --git a/test/specialfunctions/generate_data.py b/test/specialfunctions/generate_data.py new file mode 100644 index 000000000..f818c1461 --- /dev/null +++ b/test/specialfunctions/generate_data.py @@ -0,0 +1,41 @@ +import numpy as np +import torch +import os + +# create a directory to store the reference data +data_dir = 'data' +if not os.path.exists(data_dir): + os.makedirs(data_dir) + +# define a range of inputs to test +x_np = np.linspace(-10.0, 10.0, num=1001, dtype=np.float64) +x_torch = torch.tensor(x_np, requires_grad=True) + +# map the activation functions available in stdlib to their PyTorch equivalents +activations = { + 'sigmoid': torch.sigmoid, + 'tanh': torch.tanh, + 'relu': torch.relu, + 'elu': torch.nn.functional.elu, + 'selu': torch.nn.functional.selu, + 'softplus': torch.nn.functional.softplus, + 'softsign': torch.nn.functional.softsign, +} + +print('Generating reference data...') + +for name, func in activations.items(): + if x_torch.grad is not None: + x_torch.grad.zero_() + + y = func(x_torch) + y.sum().backward() + derivative = x_torch.grad.numpy() + + np.save(f'{data_dir}/{name}_input.npy', x_np) + np.save(f'{data_dir}/{name}_output.npy', y.detach().numpy()) + np.save(f'{data_dir}/{name}_derivative.npy', derivative) + + print(f'Generated data for: {name}') + +print('All reference data generated successfully.') From 74c3037332941f65a5a00f4383d2d3ea9ec308c4 Mon Sep 17 00:00:00 2001 From: learner0018 <212705726+learner0018@users.noreply.github.com> Date: Sun, 15 Mar 2026 00:25:25 +0530 Subject: [PATCH 2/6] Draft: Enhance activations testing with reference data --- .../generate_activation_data.py | 47 +++++++++++++++++++ .../test_specialfunctions_activations.fypp | 27 ++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 test/specialfunctions/generate_activation_data.py diff --git a/test/specialfunctions/generate_activation_data.py b/test/specialfunctions/generate_activation_data.py new file mode 100644 index 000000000..92d50d2db --- /dev/null +++ b/test/specialfunctions/generate_activation_data.py @@ -0,0 +1,47 @@ +# --- python script: generate_activation_data.py --- +# place this in test/specialfunctions/ + +import numpy as np +import torch +import os + +# create a directory to store the reference data +data_dir = 'data' +if not os.path.exists(data_dir): + os.makedirs(data_dir) + +# define a range of inputs to test +x_np = np.linspace(-10.0, 10.0, num=1001, dtype=np.float64) +x_torch = torch.tensor(x_np, requires_grad=True) + +# map the activation functions available in stdlib to their PyTorch equivalents +activations = { + "sigmoid": torch.sigmoid, + "tanh": torch.tanh, + "relu": torch.relu, + "elu": torch.nn.functional.elu, + "selu": torch.nn.functional.selu, + "softplus": torch.nn.functional.softplus, + "softsign": torch.nn.functional.softsign, +} + +print("Generating reference data...") + +for name, func in activations.items(): + # Reset gradients for each function + if x_torch.grad is not None: + x_torch.grad.zero_() + + # calculate the function and its derivative + y = func(x_torch) + y.sum().backward() + derivative = x_torch.grad.numpy() + + # save input, output, and derivative as .npy files + np.save(f'{data_dir}/{name}_input.npy', x_np) + np.save(f'{data_dir}/{name}_output.npy', y.detach().numpy()) + np.save(f'{data_dir}/{name}_derivative.npy', derivative) + + print(f"Generated data for: {name}") + +print("All reference data generated successfully.") \ No newline at end of file diff --git a/test/specialfunctions/test_specialfunctions_activations.fypp b/test/specialfunctions/test_specialfunctions_activations.fypp index 7ff05d3bc..24b4641ea 100644 --- a/test/specialfunctions/test_specialfunctions_activations.fypp +++ b/test/specialfunctions/test_specialfunctions_activations.fypp @@ -6,6 +6,7 @@ module test_specialfunctions_activation use stdlib_kinds use stdlib_specialfunctions use stdlib_math, only: linspace + use stdlib_io_npy_load, only: load_npy implicit none private @@ -31,7 +32,8 @@ contains new_unittest("sigmoid", test_sigmoid), & new_unittest("silu" , test_silu), & new_unittest("softmax", test_softmax), & - new_unittest("logsoftmax", test_logsoftmax) & + new_unittest("logsoftmax", test_logsoftmax), & + new_unittest("sigmoid_pytorch", test_sigmoid_pytorch) & ] end subroutine collect_specialfunctions_activation @@ -382,6 +384,29 @@ contains #:endfor end subroutine test_logsoftmax + subroutine test_sigmoid_pytorch(error) + type(error_type), allocatable, intent(out) :: error + + real(dp), allocatable :: x(:), y_ref(:), y_calc(:) + integer :: n + + ! 1. Load the data generated by Python + ! Make sure you ran 'python generate_data.py' in this folder first! + call load_npy('data/sigmoid_input.npy', x) + call load_npy('data/sigmoid_output.npy', y_ref) + + n = size(x) + allocate(y_calc(n)) + + ! 2. Calculate using stdlib + y_calc = sigmoid(x) + + ! 3. Compare + call check(error, norm2(y_calc - y_ref) < 1.0e-9_dp, & + "Sigmoid mismatch against PyTorch reference") + if (allocated(error)) return + + end subroutine test_sigmoid_pytorch end module test_specialfunctions_activation From 571ee876fdf81302888362cc5f267a9104c7128b Mon Sep 17 00:00:00 2001 From: learner0018 <212705726+learner0018@users.noreply.github.com> Date: Sun, 15 Mar 2026 11:29:15 +0530 Subject: [PATCH 3/6] Fix: Link stdlib_io dependency for activation tests --- test/specialfunctions/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/specialfunctions/CMakeLists.txt b/test/specialfunctions/CMakeLists.txt index 536c36e12..a17c4758f 100644 --- a/test/specialfunctions/CMakeLists.txt +++ b/test/specialfunctions/CMakeLists.txt @@ -9,4 +9,5 @@ set(fppFiles fypp_f90("${fyppFlags}" "${fppFiles}" outFiles) ADDTEST(specialfunctions_gamma) -ADDTEST(specialfunctions_activations) \ No newline at end of file +ADDTEST(specialfunctions_activations) +target_link_libraries(specialfunctions_activations PRIVATE stdlib_io) \ No newline at end of file From f150c7248b6121199e0d5cab0703927cbe84d92e Mon Sep 17 00:00:00 2001 From: learner0018 <212705726+learner0018@users.noreply.github.com> Date: Mon, 16 Mar 2026 23:42:12 +0530 Subject: [PATCH 4/6] Clean up: Remove unrelated app folder and generated source file --- app/test_activations.f90 | 60 -- src/stdlib_specialfunctions_activations.f90 | 761 -------------------- 2 files changed, 821 deletions(-) delete mode 100644 app/test_activations.f90 delete mode 100644 src/stdlib_specialfunctions_activations.f90 diff --git a/app/test_activations.f90 b/app/test_activations.f90 deleted file mode 100644 index 5cb5ccb78..000000000 --- a/app/test_activations.f90 +++ /dev/null @@ -1,60 +0,0 @@ -program test_activations - use iso_fortran_env, only: dp => real64 - ! We use the PARENT module, not the submodule - use stdlib_specialfunctions, only: sigmoid - implicit none - - real(dp) :: x, y, expected - integer :: pass_count - - print *, "==========================================" - print *, " STARTING ACTIVATION FUNCTION LOGIC TEST " - print *, "==========================================" - - pass_count = 0 - - ! Test 1: Sigmoid(0) should be 0.5 - x = 0.0_dp - expected = 0.5_dp - y = sigmoid(x) - - if (abs(y - expected) < 1.0e-9_dp) then - print *, "[PASS] Sigmoid(0.0) calculated correctly." - pass_count = pass_count + 1 - else - print *, "[FAIL] Sigmoid(0.0). Expected:", expected, "Got:", y - end if - - ! Test 2: Sigmoid(1) should be ~0.731 - x = 1.0_dp - expected = 0.7310585786300049_dp - y = sigmoid(x) - - if (abs(y - expected) < 1.0e-9_dp) then - print *, "[PASS] Sigmoid(1.0) calculated correctly." - pass_count = pass_count + 1 - else - print *, "[FAIL] Sigmoid(1.0). Expected:", expected, "Got:", y - end if - - ! Test 3: Sigmoid(-1) should be ~0.268 - x = -1.0_dp - expected = 0.2689414213699951_dp - y = sigmoid(x) - - if (abs(y - expected) < 1.0e-9_dp) then - print *, "[PASS] Sigmoid(-1.0) calculated correctly." - pass_count = pass_count + 1 - else - print *, "[FAIL] Sigmoid(-1.0). Expected:", expected, "Got:", y - end if - - print *, "==========================================" - if (pass_count == 3) then - print *, "ALL TESTS PASSED! (3/3)" - else - print *, "SOME TESTS FAILED:", pass_count, "/ 3" - error stop 1 - end if - -end program test_activations diff --git a/src/stdlib_specialfunctions_activations.f90 b/src/stdlib_specialfunctions_activations.f90 deleted file mode 100644 index 7cd0dfc36..000000000 --- a/src/stdlib_specialfunctions_activations.f90 +++ /dev/null @@ -1,761 +0,0 @@ -submodule(stdlib_specialfunctions) stdlib_specialfunctions_activations - use stdlib_intrinsics, only: sum => stdlib_sum - implicit none - - real(sp), parameter :: isqrt2_sp = 1._sp / sqrt(2._sp) - real(dp), parameter :: isqrt2_dp = 1._dp / sqrt(2._dp) - -contains - -!================================================== -! Gaussian -!================================================== -elemental module function gaussian_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = exp(-x**2) -end function - -elemental module function gaussian_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = -2._sp * x * exp(-x**2) -end function - -elemental module function gaussian_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = exp(-x**2) -end function - -elemental module function gaussian_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = -2._dp * x * exp(-x**2) -end function - - -!================================================== -! Exponential Linear Unit -!================================================== -elemental module function elu_sp( x , a ) result ( y ) - real(sp), intent(in) :: x - real(sp), intent(in) :: a - real(sp) :: y - y = merge( x , a * (exp(x) - 1._sp), x >= 0._sp) -end function - -elemental module function elu_grad_sp( x , a ) result ( y ) - real(sp), intent(in) :: x - real(sp), intent(in) :: a - real(sp) :: y - y = merge( 1._sp , a * exp(x), x >= 0._sp) -end function - -elemental module function elu_dp( x , a ) result ( y ) - real(dp), intent(in) :: x - real(dp), intent(in) :: a - real(dp) :: y - y = merge( x , a * (exp(x) - 1._dp), x >= 0._dp) -end function - -elemental module function elu_grad_dp( x , a ) result ( y ) - real(dp), intent(in) :: x - real(dp), intent(in) :: a - real(dp) :: y - y = merge( 1._dp , a * exp(x), x >= 0._dp) -end function - - -!================================================== -! Rectified Linear Unit -!================================================== -elemental module function relu_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = max(0._sp, x) -end function - -elemental module function relu_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = merge( 1._sp , 0._sp, x > 0._sp) -end function - -elemental module function relu_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = max(0._dp, x) -end function - -elemental module function relu_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = merge( 1._dp , 0._dp, x > 0._dp) -end function - - -!================================================== -! leaky Rectified Linear Unit -!================================================== -elemental module function leaky_relu_sp( x , a ) result( y ) - real(sp), intent(in) :: x - real(sp), intent(in) :: a - real(sp) :: y - y = merge( x, a * x , x >= 0._sp) -end function - -elemental module function leaky_relu_grad_sp( x , a ) result( y ) - real(sp), intent(in) :: x - real(sp), intent(in) :: a - real(sp) :: y - y = merge( 1._sp , a , x >= 0._sp) -end function - -elemental module function leaky_relu_dp( x , a ) result( y ) - real(dp), intent(in) :: x - real(dp), intent(in) :: a - real(dp) :: y - y = merge( x, a * x , x >= 0._dp) -end function - -elemental module function leaky_relu_grad_dp( x , a ) result( y ) - real(dp), intent(in) :: x - real(dp), intent(in) :: a - real(dp) :: y - y = merge( 1._dp , a , x >= 0._dp) -end function - - -!================================================== -! GELU: Gaussian Error Linear Units function -!================================================== -elemental module function gelu_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 0.5_sp * x * (1._sp + erf(x * isqrt2_sp)) -end function - -elemental module function gelu_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 0.5_sp * (1._sp + erf(x * isqrt2_sp) ) - y = y + x * isqrt2_sp * exp( - 0.5_sp * x**2 ) -end function - -elemental module function gelu_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 0.5_dp * x * (1._dp + erf(x * isqrt2_dp)) -end function - -elemental module function gelu_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 0.5_dp * (1._dp + erf(x * isqrt2_dp) ) - y = y + x * isqrt2_dp * exp( - 0.5_dp * x**2 ) -end function - - -elemental module function gelu_approx_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 0.5_sp * x * (1._sp + fast_erf(x * isqrt2_sp)) -end function - -elemental module function gelu_approx_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 0.5_sp * (1._sp + fast_erf(x * isqrt2_sp) ) - y = y + x * isqrt2_sp * exp( - 0.5_sp * x**2 ) -end function - -elemental module function gelu_approx_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 0.5_dp * x * (1._dp + fast_erf(x * isqrt2_dp)) -end function - -elemental module function gelu_approx_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 0.5_dp * (1._dp + fast_erf(x * isqrt2_dp) ) - y = y + x * isqrt2_dp * exp( - 0.5_dp * x**2 ) -end function - - -!================================================== -! Scaled Exponential Linear Unit (SELU) -!================================================== -elemental module function selu_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - real(sp), parameter :: scale = 1.0507009873554804934193349852946_sp - real(sp), parameter :: alpha = 1.6732632423543772848170429916717_sp - y = merge( x , alpha * exp(x) - alpha, x > 0._sp) - y = scale * y -end function - -elemental module function selu_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - real(sp), parameter :: scale = 1.0507009873554804934193349852946_sp - real(sp), parameter :: alpha = 1.6732632423543772848170429916717_sp - y = merge( scale , scale * alpha * exp(x), x > 0._sp) -end function - -elemental module function selu_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - real(dp), parameter :: scale = 1.0507009873554804934193349852946_dp - real(dp), parameter :: alpha = 1.6732632423543772848170429916717_dp - y = merge( x , alpha * exp(x) - alpha, x > 0._dp) - y = scale * y -end function - -elemental module function selu_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - real(dp), parameter :: scale = 1.0507009873554804934193349852946_dp - real(dp), parameter :: alpha = 1.6732632423543772848170429916717_dp - y = merge( scale , scale * alpha * exp(x), x > 0._dp) -end function - - -!================================================== -! Sigmoid -!================================================== -elemental module function sigmoid_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 1._sp / (1._sp + exp(-x)) -end function - -elemental module function sigmoid_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = exp(x) / (1._sp + exp(x))**2 -end function - -elemental module function sigmoid_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 1._dp / (1._dp + exp(-x)) -end function - -elemental module function sigmoid_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = exp(x) / (1._dp + exp(x))**2 -end function - - -!================================================== -! SiLU: Sigmoid Linear Unit -!================================================== -elemental module function silu_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = x / (1._sp + exp(-x)) -end function - -elemental module function silu_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = (1._sp + exp(x))**2 - y = exp(x) * ( x + y ) / y -end function - -elemental module function silu_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = x / (1._dp + exp(-x)) -end function - -elemental module function silu_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = (1._dp + exp(x))**2 - y = exp(x) * ( x + y ) / y -end function - - -!================================================== -! Step -!================================================== -elemental module function step_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = merge( 1._sp , 0._sp, x > 0._sp) -end function - -elemental module function step_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 0._sp -end function - -elemental module function step_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = merge( 1._dp , 0._dp, x > 0._dp) -end function - -elemental module function step_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 0._dp -end function - - -!================================================== -! softmax -!================================================== -pure module function softmax_r1_sp( x ) result( y ) - real(sp), intent(in) :: x(:) - real(sp) :: y(size(x)) - - y = exp(x - maxval(x)) - y = y / sum(y) -end function - -pure module function softmax_r2_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:) - real(sp) :: y(size(x,1), size(x,2)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<2)then - do j = 1, size(x,dim=2) - y(:, j) = softmax( x(:, j) ) - end do - else - do j = 1, size(x,dim=1) - y(j, :) = softmax( x(j, :) ) - end do - end if -end function -pure module function softmax_r3_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:,:) - real(sp) :: y(size(x,1), size(x,2), size(x,3)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<3)then - do j = 1, size(x,dim=3) - y(:, :, j) = softmax( x(:, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :) = softmax( x(j, :, :), dim=2 ) - end do - end if -end function -pure module function softmax_r4_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:,:,:) - real(sp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<4)then - do j = 1, size(x,dim=4) - y(:, :, :, j) = softmax( x(:, :, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :, :) = softmax( x(j, :, :, :), dim=3 ) - end do - end if -end function - -pure module function softmax_grad_r1_sp( x ) result( y ) - real(sp), intent(in) :: x(:) - real(sp) :: y(size(x)) - - y = softmax(x) - y = y * (1._sp - y) -end function - -pure module function softmax_grad_r2_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:) - real(sp) :: y(size(x,1), size(x,2)) - - integer, intent(in), optional :: dim - integer :: dim_ - - dim_ = 1; if(present(dim)) dim_ = dim - - y = softmax(x,dim_) - y = y * (1._sp - y) -end function -pure module function softmax_grad_r3_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:,:) - real(sp) :: y(size(x,1), size(x,2), size(x,3)) - - integer, intent(in), optional :: dim - integer :: dim_ - - dim_ = 1; if(present(dim)) dim_ = dim - - y = softmax(x,dim_) - y = y * (1._sp - y) -end function -pure module function softmax_grad_r4_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:,:,:) - real(sp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) - - integer, intent(in), optional :: dim - integer :: dim_ - - dim_ = 1; if(present(dim)) dim_ = dim - - y = softmax(x,dim_) - y = y * (1._sp - y) -end function - -pure module function softmax_r1_dp( x ) result( y ) - real(dp), intent(in) :: x(:) - real(dp) :: y(size(x)) - - y = exp(x - maxval(x)) - y = y / sum(y) -end function - -pure module function softmax_r2_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:) - real(dp) :: y(size(x,1), size(x,2)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<2)then - do j = 1, size(x,dim=2) - y(:, j) = softmax( x(:, j) ) - end do - else - do j = 1, size(x,dim=1) - y(j, :) = softmax( x(j, :) ) - end do - end if -end function -pure module function softmax_r3_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:,:) - real(dp) :: y(size(x,1), size(x,2), size(x,3)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<3)then - do j = 1, size(x,dim=3) - y(:, :, j) = softmax( x(:, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :) = softmax( x(j, :, :), dim=2 ) - end do - end if -end function -pure module function softmax_r4_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:,:,:) - real(dp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<4)then - do j = 1, size(x,dim=4) - y(:, :, :, j) = softmax( x(:, :, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :, :) = softmax( x(j, :, :, :), dim=3 ) - end do - end if -end function - -pure module function softmax_grad_r1_dp( x ) result( y ) - real(dp), intent(in) :: x(:) - real(dp) :: y(size(x)) - - y = softmax(x) - y = y * (1._dp - y) -end function - -pure module function softmax_grad_r2_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:) - real(dp) :: y(size(x,1), size(x,2)) - - integer, intent(in), optional :: dim - integer :: dim_ - - dim_ = 1; if(present(dim)) dim_ = dim - - y = softmax(x,dim_) - y = y * (1._dp - y) -end function -pure module function softmax_grad_r3_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:,:) - real(dp) :: y(size(x,1), size(x,2), size(x,3)) - - integer, intent(in), optional :: dim - integer :: dim_ - - dim_ = 1; if(present(dim)) dim_ = dim - - y = softmax(x,dim_) - y = y * (1._dp - y) -end function -pure module function softmax_grad_r4_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:,:,:) - real(dp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) - - integer, intent(in), optional :: dim - integer :: dim_ - - dim_ = 1; if(present(dim)) dim_ = dim - - y = softmax(x,dim_) - y = y * (1._dp - y) -end function - - -!================================================== -! logsoftmax -!================================================== -pure module function logsoftmax_r1_sp( x ) result( y ) - real(sp), intent(in) :: x(:) - real(sp) :: y(size(x)) - - y = x - maxval(x) - y = y - log( sum(exp(y)) ) -end function - -pure module function logsoftmax_r2_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:) - real(sp) :: y(size(x,1), size(x,2)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<2)then - do j = 1, size(x,dim=2) - y(:, j) = logsoftmax( x(:, j) ) - end do - else - do j = 1, size(x,dim=1) - y(j, :) = logsoftmax( x(j, :) ) - end do - end if -end function -pure module function logsoftmax_r3_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:,:) - real(sp) :: y(size(x,1), size(x,2), size(x,3)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<3)then - do j = 1, size(x,dim=3) - y(:, :, j) = logsoftmax( x(:, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :) = logsoftmax( x(j, :, :), dim=2 ) - end do - end if -end function -pure module function logsoftmax_r4_sp( x , dim ) result( y ) - real(sp), intent(in) :: x(:,:,:,:) - real(sp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<4)then - do j = 1, size(x,dim=4) - y(:, :, :, j) = logsoftmax( x(:, :, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :, :) = logsoftmax( x(j, :, :, :), dim=3 ) - end do - end if -end function - -pure module function logsoftmax_r1_dp( x ) result( y ) - real(dp), intent(in) :: x(:) - real(dp) :: y(size(x)) - - y = x - maxval(x) - y = y - log( sum(exp(y)) ) -end function - -pure module function logsoftmax_r2_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:) - real(dp) :: y(size(x,1), size(x,2)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<2)then - do j = 1, size(x,dim=2) - y(:, j) = logsoftmax( x(:, j) ) - end do - else - do j = 1, size(x,dim=1) - y(j, :) = logsoftmax( x(j, :) ) - end do - end if -end function -pure module function logsoftmax_r3_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:,:) - real(dp) :: y(size(x,1), size(x,2), size(x,3)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<3)then - do j = 1, size(x,dim=3) - y(:, :, j) = logsoftmax( x(:, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :) = logsoftmax( x(j, :, :), dim=2 ) - end do - end if -end function -pure module function logsoftmax_r4_dp( x , dim ) result( y ) - real(dp), intent(in) :: x(:,:,:,:) - real(dp) :: y(size(x,1), size(x,2), size(x,3), size(x,4)) - - integer, intent(in), optional :: dim - integer :: dim_, j - - dim_ = 1; if(present(dim)) dim_ = dim - - if(dim_<4)then - do j = 1, size(x,dim=4) - y(:, :, :, j) = logsoftmax( x(:, :, :, j), dim=dim_ ) - end do - else - do j = 1, size(x,dim=1) - y(j, :, :, :) = logsoftmax( x(j, :, :, :), dim=3 ) - end do - end if -end function - - -!================================================== -! softplus -!================================================== -elemental module function softplus_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = log(exp(x) + 1._sp) -end function - -elemental module function softplus_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = exp(x) / (exp(x) + 1._sp) -end function - -elemental module function softplus_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = log(exp(x) + 1._dp) -end function - -elemental module function softplus_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = exp(x) / (exp(x) + 1._dp) -end function - - -!================================================== -! Fast intrinsics for accelerated activations -!================================================== - -! Source: https://fortran-lang.discourse.group/t/fastgpt-faster-than-pytorch-in-300-lines-of-fortran/5385/31 -elemental module function fast_tanh_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - real(sp) :: x2, a, b - - x2 = x*x - a = x * (135135.0_sp + x2 * (17325.0_sp + x2 * (378.0_sp + x2))) - b = 135135.0_sp + x2 * (62370.0_sp + x2 * (3150.0_sp + x2 * 28.0_sp)) - y = merge( a / b , sign(1._sp,x) , x2 <= 25._sp ) -end function - -elemental module function fast_tanh_grad_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - y = 1._sp - fast_tanh(x)**2 -end function - -elemental module function fast_erf_sp( x ) result( y ) - real(sp), intent(in) :: x - real(sp) :: y - real(sp) :: abs_x - - abs_x = abs(x) - y = 1._sp - 1._sp / (1._sp+ 0.278393_sp*abs_x + 0.230389_sp*abs_x**2 + 0.000972_sp*abs_x**3 + 0.078108_sp*abs_x**4)**4 - y = y * sign(1.0_sp,x) -end function - -elemental module function fast_tanh_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - real(dp) :: x2, a, b - - x2 = x*x - a = x * (135135.0_dp + x2 * (17325.0_dp + x2 * (378.0_dp + x2))) - b = 135135.0_dp + x2 * (62370.0_dp + x2 * (3150.0_dp + x2 * 28.0_dp)) - y = merge( a / b , sign(1._dp,x) , x2 <= 25._dp ) -end function - -elemental module function fast_tanh_grad_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - y = 1._dp - fast_tanh(x)**2 -end function - -elemental module function fast_erf_dp( x ) result( y ) - real(dp), intent(in) :: x - real(dp) :: y - real(dp) :: abs_x - - abs_x = abs(x) - y = 1._dp - 1._dp / (1._dp+ 0.278393_dp*abs_x + 0.230389_dp*abs_x**2 + 0.000972_dp*abs_x**3 + 0.078108_dp*abs_x**4)**4 - y = y * sign(1.0_dp,x) -end function - - -end submodule \ No newline at end of file From 706d50bcb42f8b6ac92c8cbddde21669ab651172 Mon Sep 17 00:00:00 2001 From: learner0018 <212705726+learner0018@users.noreply.github.com> Date: Mon, 16 Mar 2026 23:47:29 +0530 Subject: [PATCH 5/6] Clean up: Remove duplicate script and fix CMake --- test/specialfunctions/generate_data.py | 41 -------------------------- 1 file changed, 41 deletions(-) delete mode 100644 test/specialfunctions/generate_data.py diff --git a/test/specialfunctions/generate_data.py b/test/specialfunctions/generate_data.py deleted file mode 100644 index f818c1461..000000000 --- a/test/specialfunctions/generate_data.py +++ /dev/null @@ -1,41 +0,0 @@ -import numpy as np -import torch -import os - -# create a directory to store the reference data -data_dir = 'data' -if not os.path.exists(data_dir): - os.makedirs(data_dir) - -# define a range of inputs to test -x_np = np.linspace(-10.0, 10.0, num=1001, dtype=np.float64) -x_torch = torch.tensor(x_np, requires_grad=True) - -# map the activation functions available in stdlib to their PyTorch equivalents -activations = { - 'sigmoid': torch.sigmoid, - 'tanh': torch.tanh, - 'relu': torch.relu, - 'elu': torch.nn.functional.elu, - 'selu': torch.nn.functional.selu, - 'softplus': torch.nn.functional.softplus, - 'softsign': torch.nn.functional.softsign, -} - -print('Generating reference data...') - -for name, func in activations.items(): - if x_torch.grad is not None: - x_torch.grad.zero_() - - y = func(x_torch) - y.sum().backward() - derivative = x_torch.grad.numpy() - - np.save(f'{data_dir}/{name}_input.npy', x_np) - np.save(f'{data_dir}/{name}_output.npy', y.detach().numpy()) - np.save(f'{data_dir}/{name}_derivative.npy', derivative) - - print(f'Generated data for: {name}') - -print('All reference data generated successfully.') From 3427f1cac09434b806baefe8f28e93346b08c245 Mon Sep 17 00:00:00 2001 From: learner0018 <212705726+learner0018@users.noreply.github.com> Date: Mon, 16 Mar 2026 23:48:43 +0530 Subject: [PATCH 6/6] Clean up: Remove duplicate script and fix CMake --- test/specialfunctions/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/specialfunctions/CMakeLists.txt b/test/specialfunctions/CMakeLists.txt index a17c4758f..60eefea71 100644 --- a/test/specialfunctions/CMakeLists.txt +++ b/test/specialfunctions/CMakeLists.txt @@ -10,4 +10,5 @@ fypp_f90("${fyppFlags}" "${fppFiles}" outFiles) ADDTEST(specialfunctions_gamma) ADDTEST(specialfunctions_activations) + target_link_libraries(specialfunctions_activations PRIVATE stdlib_io) \ No newline at end of file