diff --git a/bin/compile b/bin/compile index 6aefdb280..5a41fef39 100755 --- a/bin/compile +++ b/bin/compile @@ -95,6 +95,14 @@ export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/. export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config +# Add apt-related paths +export LD_LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LD_LIBRARY_PATH" +export LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LIBRARY_PATH" +export INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$INCLUDE_PATH" +export C_INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$C_INCLUDE_PATH" +export CPLUS_INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$CPLUS_INCLUDE_PATH" +export PKG_CONFIG_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/pkgconfig:$PKG_CONFIG_PATH" + # Switch to the repo's context. cd $BUILD_DIR @@ -126,13 +134,15 @@ if [ ! -f runtime.txt ]; then fi # ### The Cache +puts-step "Creating cache directory in $CACHE_DIR" mkdir -p $CACHE_DIR +ls -l $CACHE_DIR # Purge "old-style" virtualenvs. -bpwatch start clear_old_venvs - [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include - [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src -bpwatch stop clear_old_venvs +#bpwatch start clear_old_venvs +# [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include +# [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src +#bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache @@ -166,22 +176,29 @@ source $BIN_DIR/steps/pylibmc # Libffi support. source $BIN_DIR/steps/cryptography -# Install dependencies with Pip. -sub-env $BIN_DIR/steps/pip-install +# Numpy/scipy support +source $BIN_DIR/steps/numpy-scipy -# Django collectstatic support. -sub-env $BIN_DIR/steps/collectstatic +# Force scikit-learn to use a specific script +source $BIN_DIR/steps/scikit-learn +# Install dependencies with Pip. +sub-env $BIN_DIR/steps/pip-install # ### Finalize # +export LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib + +puts-step "Setting environment variables (LD_LIBRARY_PATH = $LD_LIBRARY_PATH)" + # Set context environment variables. set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true set-env PYTHONHOME /app/.heroku/python -set-env LIBRARY_PATH /app/.heroku/vendor/lib:/app/.heroku/python/lib -set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH' +set-env LIBRARY_PATH /app/.heroku/vendor/lib:/app/.heroku/python/lib:/app/vendor/libffi-3.0/lib +set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:/app/vendor/libffi-3.0/lib:$LD_LIBRARY_PATH' set-default-env LANG en_US.UTF-8 set-default-env PYTHONHASHSEED random set-default-env PYTHONPATH /app/ diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy new file mode 100644 index 000000000..3d5434d7a --- /dev/null +++ b/bin/steps/numpy-scipy @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# This script serves as the Scipy/Numpy build step of the +# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) +# compiler. +# +# A [buildpack](http://devcenter.heroku.com/articles/buildpacks) is an +# adapter between a Python application and Heroku's runtime. +# +# This script is invoked by [`bin/compile`](/). + +VENDORED_GIT_NAME="npscipy-binaries" +VENDORED_GIT_REPO="git://github.com/wyn/${VENDORED_GIT_NAME}.git" +# The location of the pre-compiled atlas/lapack/blas/gfortran binaries. +VENDORED_BINARIES="npscipy" +# the location of the bdist of numpy +VENDORED_NUMPY="numpy-1.7.0" +# the location of the bdist of scipy +VENDORED_SCIPY="scipy-0.11.0" + +# Syntax sugar. +source $BIN_DIR/utils + +# If numpy or scipy exists within requirements.txt then use binaries. +if (grep -iq -e "numpy" -e "scipy" requirements.txt) then + puts-step "Noticed numpy/scipy. Bootstrapping prebuilt binaries." + cd .heroku + + ## check whether this has happened already + ## set BLAS/LAPACK/ATLAS + if [ -d "vendor/lib/atlas-base" ]; then + puts-step "Using cached binaries." + else + puts-step "Creating/downloading binaries." + # Download and extract everything into target vendor directory. + git clone ${VENDORED_GIT_REPO} > /dev/null + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_BINARIES}.tar.gz > /dev/null + fi + export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a + export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a + export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a + export LIBRARY_PATH=$LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + + if [ -d "python/lib/python2.7/site-packages/numpy" ]; then + puts-step "Using cached numpy." + else + puts-step "Creating/downloading numpy bdist." + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_NUMPY}.tar.gz > /dev/null + fi + + if [ -d "python/lib/python2.7/site-packages/scipy" ]; then + puts-step "Using cached scipy." + else + puts-step "Creating/downloading scipy bdist." + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_SCIPY}.tar.gz > /dev/null + fi + + # Move everything from venv directory to python directory + if [ -d "venv" ]; then + cp -a venv/* python/ + rm -r venv + fi + + if [ -d ${VENDORED_GIT_NAME} ]; then + rm -rf ${VENDORED_GIT_NAME} + fi + cd .. +fi diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 05b97ab78..356179f4c 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -5,6 +5,9 @@ source $BIN_DIR/utils # Install dependencies with Pip. puts-step "Installing dependencies with pip" +echo $LIBRARY_PATH +echo $INCLUDE_PATH + [ ! "$FRESH_PYTHON" ] && bpwatch start pip_install [ "$FRESH_PYTHON" ] && bpwatch start pip_install_first diff --git a/bin/steps/scikit-learn b/bin/steps/scikit-learn new file mode 100644 index 000000000..c6ec9b125 --- /dev/null +++ b/bin/steps/scikit-learn @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# This script serves as the Skikit-learn build step of the +# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) +# compiler. +# +# A [buildpack](http://devcenter.heroku.com/articles/buildpacks) is an +# adapter between a Python application and Heroku's runtime. +# +# This script is invoked by [`bin/compile`](/). + +# Syntax sugar. +source $BIN_DIR/utils + +# If skikit-learn exists within requirements.txt then use binaries. +if (grep -iq -e "^\s*scikit-learn" requirements.txt) then + TARGET_SKLEARN_VERSION=$(cat requirements.txt | grep scikit-learn | tr -d " " | cut -c 13-) + puts-step "Noticed scikit-learn. Installing..." + cd .heroku + + ## check whether this has happened already + + if [ -d "python/lib/python2.7/site-packages/scikit-learn*" ]; then + puts-step "Using cached scikit-learn." + else + puts-step "Installing scikit-learn$TARGET_SKLEARN_VERSION via pip, but setting environment vars first." + export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a + export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a + export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a + export LIBRARY_PATH=$LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + + pip install --use-mirrors scikit-learn$TARGET_SKLEARN_VERSION + fi + + cd .. + +fi