ENH limit threads for C-Libraries dynamically

.gitignore

@@ -15,6 +15,3 @@ __pycache__
 /dist
 /htmlcov
 
-# Generated by Cython test
-tests/_openmp/build/
-tests/_openmp/parallel_sum.c

.travis.yml

@@ -21,15 +21,15 @@ matrix:
     - os: osx
       osx_image: xcode7.3
       language: generic  # https://github.com/travis-ci/travis-ci/issues/2312
-      env: TOXENV="py37" PYTHON="python3" RUN_MEMORY="true"
+      env: TOXENV="py37" PYTHON="python3" RUN_MEMORY="true" LOKY_TEST_FORCE_OPENBLAS="true"
     - python: 2.7
       env: TOXENV="py27"
     - python: 3.4
       env: TOXENV="py34"
     - python: 3.5
       env: TOXENV="py35"
     - python: "3.6"
-      env: TOXENV="py36"
+      env: TOXENV="py36" LOKY_TEST_FORCE_OPENBLAS="true"
     - python: "3.7-dev"
       env: TOXENV="py37" RUN_MEMORY="true"
     - python: "3.6"

appveyor.yml

@@ -3,6 +3,8 @@ environment:
   # Python versions sequentially.
   # We run it once per architecture (32 bit vs 64 bit).
   matrix:
+    - TEST_CONDA: "true"
+
     - PYTHON: "C:\\Python36"
       PYTHON_VERSION: "3.6.x"
       PYTHON_ARCH: "32"
@@ -12,6 +14,7 @@ environment:
       PYTHON_VERSION: "3.6.x"
       PYTHON_ARCH: "64"
       PYTHON_ARCH_SUFFIX: "-x64"
+
 matrix:
   fast_finish: true
 
@@ -22,9 +25,7 @@ install:
   # Install Python (from the official .msi of http://python.org) and pip when
   # not already installed.
   - ps: ./continuous_integration/appveyor/install.ps1
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "SET LOKY_MAX_DEPTH=3"
-  - "pip install tox"
 
 # Not a .NET project, we build in the install step instead
 build: false

continuous_integration/appveyor/install.ps1

@@ -7,6 +7,8 @@ $BASE_URL = "https://www.python.org/ftp/python/"
 $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
 $GET_PIP_PATH = "C:\get-pip.py"
 
+$env:VIRTUALENV = "test_env"
+
 
 function DownloadPython ($python_version, $platform_suffix) {
     $webclient = New-Object System.Net.WebClient
@@ -93,88 +95,44 @@ function InstallPip ($python_home) {
     }
 }
 
+function InstallConda (){
 
-function DownloadMiniconda ($python_version, $platform_suffix) {
-    $webclient = New-Object System.Net.WebClient
-    if ($python_version -eq "3.4") {
-        $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
-    } else {
-        $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
-    }
-    $url = $MINICONDA_URL + $filename
+    conda update -y -q conda
+    conda init powershell
+    . C:\Users\appveyor\Documents\PowerShell\profile.ps1
 
-    $basedir = $pwd.Path + "\"
-    $filepath = $basedir + $filename
-    if (Test-Path $filename) {
-        Write-Host "Reusing" $filepath
-        return $filepath
-    }
+    # Clean all previous environment that might exists
+    conda remove --all -q -y -n $env:VIRTUALENV
+    conda create -n $env:VIRTUALENV -q -y python numpy cython pytest psutil
 
-    # Download and retry up to 3 times in case of network transient errors.
-    Write-Host "Downloading" $filename "from" $url
-    $retry_attempts = 2
-    for($i=0; $i -lt $retry_attempts; $i++){
-        try {
-            $webclient.DownloadFile($url, $filepath)
-            break
-        }
-        Catch [Exception]{
-            Start-Sleep 1
-        }
-   }
-   if (Test-Path $filepath) {
-       Write-Host "File saved at" $filepath
-   } else {
-       # Retry once to get the error message if any at the last try
-       $webclient.DownloadFile($url, $filepath)
-   }
-   return $filepath
-}
+    # Activate the envrionment
+    conda activate $env:VIRTUALENV
 
+    # Print the information on the conda env
+    conda info
 
-function InstallMiniconda ($python_version, $architecture, $python_home) {
-    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
-    if (Test-Path $python_home) {
-        Write-Host $python_home "already exists, skipping."
-        return $false
-    }
-    if ($architecture -eq "32") {
-        $platform_suffix = "x86"
-    } else {
-        $platform_suffix = "x86_64"
-    }
-    $filepath = DownloadMiniconda $python_version $platform_suffix
-    Write-Host "Installing" $filepath "to" $python_home
-    $install_log = $python_home + ".log"
-    $args = "/S /D=$python_home"
-    Write-Host $filepath $args
-    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
-    if (Test-Path $python_home) {
-        Write-Host "Python $python_version ($architecture) installation complete"
-    } else {
-        Write-Host "Failed to install Python in $python_home"
-        Get-Content -Path $install_log
-        Exit 1
-    }
-}
+    # Install test dependencies and loky
+    pip install pytest-timeout
+    pip install .
 
-
-function InstallMinicondaPip ($python_home) {
-    $pip_path = $python_home + "\Scripts\pip.exe"
-    $conda_path = $python_home + "\Scripts\conda.exe"
-    if (-not(Test-Path $pip_path)) {
-        Write-Host "Installing pip..."
-        $args = "install --yes pip"
-        Write-Host $conda_path $args
-        Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
-    } else {
-        Write-Host "pip already installed."
-    }
+    # Build external test dependency
+    bash ./continuous_integration/build_test_ext.sh
 }
 
+
 function main () {
-    InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
-    InstallPip $env:PYTHON
+    If( $env:TEST_CONDA -eq "true" ){
+        $env:PATH = "C:\\Miniconda3-x64;C:\\Miniconda3-x64\\Scripts;$env:PATH"
+        InstallConda
+    }Else{
+        InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
+        InstallPip $env:PYTHON
+
+
+        $env:PATH = "$PYTHON;$PYTHON\\Scripts;$env:PATH"
+
+        pip install tox
+    }
 }
 
 main

continuous_integration/appveyor/runtests.ps1

@@ -5,15 +5,19 @@
 
 $VERSION=(36, 27)
 $TOX_CMD = "python ./continuous_integration/appveyor/tox"
+$DEFAULT_PYTEST_ARGS = "-vlx --timeout=50 --skip-high-memory"
 
-function TestPythonVersions () {
+function RunTestsWithTox () {
     Write-Host $PYTHON
     ForEach($ver in $VERSION){
         $env:TOXPYTHON = "C:\Python$ver$env:PYTHON_ARCH_SUFFIX\python.exe"
         Write-Host $env:TOXPYTHON
         # Skip memory test as the appveyor environment is too small for those.
-        $PYTEST_ARGS = "-vl --timeout=60 --maxfail=5 --skip-high-memory"
+        $PYTEST_ARGS = $DEFAULT_PYTEST_ARGS
 
+        If( $ver -eq 36){
+            $PYTEST_ARGS = "$PYTEST_ARGS --openblas-test-noskip"
+        }
         # Launch the tox command for the correct python version. We use `iex`
         # to correctly pass PYTEST_ARGS, which are parsed as files otherwise.
         iex "$TOX_CMD -e py$ver -- $PYTEST_ARGS"
@@ -24,4 +28,31 @@ function TestPythonVersions () {
     Exit 0
 }
 
-TestPythonVersions
+
+function RunTestsWithConda () {
+
+    # Activate the envrionment
+    conda activate $env:VIRTUALENV
+
+    # test numpy installation
+    python --version
+    python -c "import numpy"
+
+    iex "pytest $DEFAULT_PYTEST_ARGS --mkl-win32-test-noskip"
+    If( $LASTEXITCODE -ne 0){
+            Exit 1
+    }
+    Exit 0
+
+}
+
+function main () {
+    If( $env:TEST_CONDA -eq "true" ){
+        RunTestsWithConda
+    }
+    Else{
+        RunTestsWithTox
+    }
+}
+
+main

continuous_integration/build_test_ext.sh

@@ -1,5 +1,5 @@
 
-cd tests/_openmp
+cd tests/_openmp_test_helper
 COVERAGE_PROCESS_START=''
 python setup.py build_ext -i || echo 'No openmp'
-cd ../..
+cd ../..

continuous_integration/travis/runtests.sh

@@ -27,11 +27,20 @@ if [ "$JOBLIB_TESTS" = "true" ]; then
     (cd $JOBLIB/externals && bash copy_loky.sh $TRAVIS_BUILD_DIR)
     pytest -vl --ignore $JOBLIB/externals --pyargs joblib
 else
-    # Run the tests and collect trace coverage data both in the subprocesses
-    # and its subprocesses.
-    PYTEST_ARGS="-vl --timeout=60 --maxfail=5"
+
+    export PYTEST_ARGS="-vl --timeout=60 --maxfail=5"
+    if [ "$LOKY_TEST_FORCE_OPENBLAS" = "true" ]; then
+        # Fail if OpenBlas is not found. This should be used when numpy is
+        # installed via pip.
+        export PYTEST_ARGS="$PYTEST_ARGS --openblas-test-noskip"
+    fi
     if [ "$RUN_MEMORY" != "true" ]; then
+        # Skip high memory usage tests when there are not required on CI
         PYTEST_ARGS="$PYTEST_ARGS --skip-high-memory"
     fi
+
+    # Run the tests and collect trace coverage data both in the subprocesses
+    # and its subprocesses.
     COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" $PYTHON -m tox -- $PYTEST_ARGS
+
 fi

docs/API.rst

@@ -1,9 +1,10 @@
 API Reference
 =============
 
-
 .. automodule:: loky
-    :members: get_reusable_executor
+    :members: get_reusable_executor, cpu_count
+
+.. autofunction:: loky.backend.thread_pool_limits
 
 
 Task & results serialization
@@ -37,16 +38,16 @@ Protection against memory leaks
 The memory size of long running worker processes can increase indefinitely if a
 memory leak is created. This can result in processes being shut down by the OS if
 those leaks are not resolved. To
-prevent it, loky provides leak detection, memory cleanups, and workers 
+prevent it, loky provides leak detection, memory cleanups, and workers
 shutdown.
 
-If :mod:`psutil` is installed, each worker periodically [#periodically_fn]_ checks its 
+If :mod:`psutil` is installed, each worker periodically [#periodically_fn]_ checks its
 memory usage after it completes its task. If the usage is found to be
-unusual [#psutil_unusual_fn]_, an additional :code:`gc.collect()` event is triggered to remove 
-objects with potential cyclic references. 
-If even after that, the memory usage of a process worker remains too high, 
+unusual [#psutil_unusual_fn]_, an additional :code:`gc.collect()` event is triggered to remove
+objects with potential cyclic references.
+If even after that, the memory usage of a process worker remains too high,
 it will shut down safely, and a fresh process will be automatically spawned by
-the executor. 
+the executor.
 
 If :mod:`psutil` is not installed, there is no easy way to monitor worker
 processes memory usage. :code:`gc.collect()` events will still be called

loky/backend/__init__.py

@@ -2,6 +2,7 @@
 import sys
 
 from .context import get_context
+from ._thread_pool_limiters import thread_pool_limits
 
 if sys.version_info > (3, 4):
 
@@ -13,4 +14,4 @@ def _make_name():
     from multiprocessing import synchronize
     synchronize.SemLock._make_name = staticmethod(_make_name)
 
-__all__ = ["get_context"]
+__all__ = ["get_context", "thread_pool_limits"]