From 621773ce22e7a20206307c88df98b1a715401d0d Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 3 Dec 2013 21:59:59 +0000 Subject: [PATCH 01/33] Remove collectstatic (ran separately) --- bin/compile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 01ae6aa69..77374835c 100755 --- a/bin/compile +++ b/bin/compile @@ -221,9 +221,9 @@ puts-step "Installing dependencies using Pip ($PIP_VERSION)" [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first # Django collectstatic support. -bpwatch start collectstatic - source $BIN_DIR/steps/collectstatic -bpwatch stop collectstatic +#bpwatch start collectstatic +# source $BIN_DIR/steps/collectstatic +#bpwatch stop collectstatic # ### Finalize # From 76a5d970e7d4231c89c8abb6064802e82e30e867 Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:13:03 -0600 Subject: [PATCH 02/33] Create numpy-scipy https://github.com/dbrgn/heroku-buildpack-python-sklearn --- bin/steps/numpy-scipy | 69 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 bin/steps/numpy-scipy diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy new file mode 100644 index 000000000..a5bf7d5fb --- /dev/null +++ b/bin/steps/numpy-scipy @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# This script serves as the Scipy/Numpy build step of the +# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) +# compiler. +# +# A [buildpack](http://devcenter.heroku.com/articles/buildpacks) is an +# adapter between a Python application and Heroku's runtime. +# +# This script is invoked by [`bin/compile`](/). + +VENDORED_GIT_NAME="npscipy-binaries" +VENDORED_GIT_REPO="git://github.com/wyn/${VENDORED_GIT_NAME}.git" +# The location of the pre-compiled atlas/lapack/blas/gfortran binaries. +VENDORED_BINARIES="npscipy" +# the location of the bdist of numpy +VENDORED_NUMPY="numpy-1.7.0" +# the location of the bdist of scipy +VENDORED_SCIPY="scipy-0.11.0" + +# Syntax sugar. +source $BIN_DIR/utils + +# If numpy or scipy exists within requirements.txt then use binaries. +if (grep -iq -e "numpy" -e "scipy" requirements.txt) then + puts-step "Noticed numpy/scipy. Bootstrapping prebuilt binaries." + cd .heroku + + ## check whether this has happened already + ## set BLAS/LAPACK/ATLAS + if [ -d "vendor/lib/atlas-base" ]; then + puts-step "Using cached binaries." + else + puts-step "Creating/downloading binaries." + # Download and extract everything into target vendor directory. + git clone ${VENDORED_GIT_REPO} > /dev/null + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_BINARIES}.tar.gz > /dev/null + fi + export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a + export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a + export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a + export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + + if [ -d "python/lib/python2.7/site-packages/numpy" ]; then + puts-step "Using cached numpy." + else + puts-step "Creating/downloading numpy bdist." + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_NUMPY}.tar.gz > /dev/null + fi + + if [ -d "python/lib/python2.7/site-packages/scipy" ]; then + puts-step "Using cached scipy." + else + puts-step "Creating/downloading scipy bdist." + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_SCIPY}.tar.gz > /dev/null + fi + + # Move everything from venv directory to python directory + if [ -d "venv" ]; then + rsync -q -a venv/ python/ + rm -r venv + fi + + if [ -d ${VENDORED_GIT_NAME} ]; then + rm -rf ${VENDORED_GIT_NAME} + fi + cd .. +fi From d79243d421abb992ad725e8c4144f0ff16eb20e2 Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:14:33 -0600 Subject: [PATCH 03/33] Create scikit-learn From https://github.com/dbrgn/heroku-buildpack-python-sklearn --- bin/steps/scikit-learn | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 bin/steps/scikit-learn diff --git a/bin/steps/scikit-learn b/bin/steps/scikit-learn new file mode 100644 index 000000000..c19faa5ee --- /dev/null +++ b/bin/steps/scikit-learn @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# This script serves as the Skikit-learn build step of the +# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) +# compiler. +# +# A [buildpack](http://devcenter.heroku.com/articles/buildpacks) is an +# adapter between a Python application and Heroku's runtime. +# +# This script is invoked by [`bin/compile`](/). + +# Syntax sugar. +source $BIN_DIR/utils + +# If skikit-learn exists within requirements.txt then use binaries. +if (grep -iq -e "^\s*scikit-learn" requirements.txt) then + TARGET_SKLEARN_VERSION=$(cat requirements.txt | grep scikit-learn | tr -d " " | cut -c 13-) + puts-step "Noticed scikit-learn. Installing..." + cd .heroku + + ## check whether this has happened already + + if [ -d "python/lib/python2.7/site-packages/scikit-learn*" ]; then + puts-step "Using cached scikit-learn." + else + puts-step "Installing scikit-learn$TARGET_SKLEARN_VERSION via pip, but setting environment vars first." + export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a + export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a + export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a + export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + + pip install --use-mirrors scikit-learn$TARGET_SKLEARN_VERSION + fi + + cd .. + +fi From 1176431a392d99e41b33086b456efc5e2acada3a Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:15:59 -0600 Subject: [PATCH 04/33] Add numpy and scikit-learn From https://github.com/dbrgn/heroku-buildpack-python-sklearn/commit/87cf7b24a358b916deaf26b784ea95be42590efe --- bin/compile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/compile b/bin/compile index 77374835c..662f53236 100755 --- a/bin/compile +++ b/bin/compile @@ -201,6 +201,9 @@ bpwatch start pylibmc_install source $BIN_DIR/steps/pylibmc bpwatch stop pylibmc_install +# Numpy/scipy support +source $BIN_DIR/steps/numpy-scipy + # Install Mercurial if it appears to be required. if (grep -Fiq "hg+" requirements.txt) then bpwatch start mercurial_install @@ -211,6 +214,10 @@ fi # Install dependencies with Pip. puts-step "Installing dependencies using Pip ($PIP_VERSION)" +# Force scikit-learn to use a specific script +source $BIN_DIR/steps/scikit-learn + +# Actually install requirements [ ! "$FRESH_PYTHON" ] && bpwatch start pip_install [ "$FRESH_PYTHON" ] && bpwatch start pip_install_first From f0237f6197545820d5c598bf6888e0bee17f315d Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:29:03 -0600 Subject: [PATCH 05/33] Replaced rsync with cp --- bin/steps/numpy-scipy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy index a5bf7d5fb..c8f530e8c 100644 --- a/bin/steps/numpy-scipy +++ b/bin/steps/numpy-scipy @@ -58,7 +58,7 @@ if (grep -iq -e "numpy" -e "scipy" requirements.txt) then # Move everything from venv directory to python directory if [ -d "venv" ]; then - rsync -q -a venv/ python/ + cp -a venv/* python/ rm -r venv fi From 4304d77b7a26ee619be8c2bca4a53e8cf0965503 Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:55:25 -0600 Subject: [PATCH 06/33] Update compile --- bin/compile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 662f53236..d13482fb2 100755 --- a/bin/compile +++ b/bin/compile @@ -85,7 +85,7 @@ export LANG=en_US.UTF-8 export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib -export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib +export LD_LIBRARY_PATH=$LIBRARY_PATH export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config # Switch to the repo's context. @@ -239,8 +239,8 @@ source $BIN_DIR/steps/scikit-learn set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true set-env PYTHONHOME /app/.heroku/python -set-default-env LIBRARY_PATH /app/.heroku/vendor/lib -set-default-env LD_LIBRARY_PATH /app/.heroku/vendor/lib +set-default-env LIBRARY_PATH $LIBRARY_PATH +set-default-env LD_LIBRARY_PATH $LD_LIBRARY_PATH set-default-env LANG en_US.UTF-8 set-default-env PYTHONHASHSEED random set-default-env PYTHONPATH /app/ From abc21f6e8eb7afe1059ae4bd5574895d87231a3e Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:39:04 -0500 Subject: [PATCH 07/33] Add libffi step https://raw.githubusercontent.com/mfenniak/heroku-buildpack-python-libffi/master/bin/steps/libffi --- bin/steps/libffi | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 bin/steps/libffi diff --git a/bin/steps/libffi b/bin/steps/libffi new file mode 100644 index 000000000..9c11d055f --- /dev/null +++ b/bin/steps/libffi @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# This script is invoked by [`bin/compile`](/). + +# The location of the pre-compiled libffi binary. +VENDORED_LIBFFI="https://s3-us-west-2.amazonaws.com/mfenniak-graphviz/libffi-3.0.tgz" +VENDORED_PATH=/app/vendor/libffi-3.0 + +# Syntax sugar. +source $BIN_DIR/utils + +# Use vendored libffi. +echo "-----> Noticed cffi. Bootstrapping libffi." +PREV_DIR=$(pwd) +mkdir -p "$VENDORED_PATH" +cd "$VENDORED_PATH" + +# Download and extract libffi into target vendor directory. +curl -s -L -o tmp-libffi.tar.gz $VENDORED_LIBFFI +tar -zxvf tmp-libffi.tar.gz > /dev/null +rm tmp-libffi.tar.gz + +# Add libffi's to PKG_CONFIG_PATH +export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/lib/pkgconfig/ +echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" +if [ -e $(pwd)/lib/pkgconfig/libffi.pc ] +then + echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" +fi + +cd "$PREV_DIR" + + + From 453c9867833b54304bc6f47a2f1c33ddb997475d Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:41:06 -0500 Subject: [PATCH 08/33] Update compile --- bin/compile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index d13482fb2..ef19fcebb 100755 --- a/bin/compile +++ b/bin/compile @@ -85,7 +85,7 @@ export LANG=en_US.UTF-8 export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib -export LD_LIBRARY_PATH=$LIBRARY_PATH +export LD_LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config # Switch to the repo's context. @@ -203,6 +203,9 @@ bpwatch stop pylibmc_install # Numpy/scipy support source $BIN_DIR/steps/numpy-scipy + +# libffi, needed for the cryptography library +source $BIN_DIR/steps/libffi # Install Mercurial if it appears to be required. if (grep -Fiq "hg+" requirements.txt) then From a9583c0579ae13244e91bb9dcc9e3959d34d99ae Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:47:48 -0500 Subject: [PATCH 09/33] Update libffi --- bin/steps/libffi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/steps/libffi b/bin/steps/libffi index 9c11d055f..39a2ad0a0 100644 --- a/bin/steps/libffi +++ b/bin/steps/libffi @@ -28,6 +28,8 @@ then echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" fi +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ + cd "$PREV_DIR" From 3e63aeb2f16ee64d0ead8891b879959014e2cdce Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:49:18 -0500 Subject: [PATCH 10/33] Update libffi --- bin/steps/libffi | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/steps/libffi b/bin/steps/libffi index 39a2ad0a0..aec08bf73 100644 --- a/bin/steps/libffi +++ b/bin/steps/libffi @@ -3,7 +3,7 @@ # This script is invoked by [`bin/compile`](/). # The location of the pre-compiled libffi binary. -VENDORED_LIBFFI="https://s3-us-west-2.amazonaws.com/mfenniak-graphviz/libffi-3.0.tgz" +VENDORED_LIBFFI="https://s3.amazonaws.com/rt-uploads/libffi-3.0.tgz" VENDORED_PATH=/app/vendor/libffi-3.0 # Syntax sugar. @@ -22,15 +22,11 @@ rm tmp-libffi.tar.gz # Add libffi's to PKG_CONFIG_PATH export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/lib/pkgconfig/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" if [ -e $(pwd)/lib/pkgconfig/libffi.pc ] then echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" fi -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ - cd "$PREV_DIR" - - - From 6d13f6453cea5307c6f52b1054b016926c1e2acf Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:55:32 -0500 Subject: [PATCH 11/33] Don --- bin/steps/numpy-scipy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy index c8f530e8c..3d5434d7a 100644 --- a/bin/steps/numpy-scipy +++ b/bin/steps/numpy-scipy @@ -39,8 +39,8 @@ if (grep -iq -e "numpy" -e "scipy" requirements.txt) then export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a - export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas - export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LIBRARY_PATH=$LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas if [ -d "python/lib/python2.7/site-packages/numpy" ]; then puts-step "Using cached numpy." From 01f16e50452ccd5d566eca8611ee2af16ee6da62 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:56:31 -0500 Subject: [PATCH 12/33] Don't overwrite *_LIBRARY_PATH --- bin/steps/scikit-learn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/steps/scikit-learn b/bin/steps/scikit-learn index c19faa5ee..c6ec9b125 100644 --- a/bin/steps/scikit-learn +++ b/bin/steps/scikit-learn @@ -27,8 +27,8 @@ if (grep -iq -e "^\s*scikit-learn" requirements.txt) then export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a - export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas - export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LIBRARY_PATH=$LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas pip install --use-mirrors scikit-learn$TARGET_SKLEARN_VERSION fi From f71606fcb6366c4224807c1aa8b091849e01c336 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 14:25:55 -0500 Subject: [PATCH 13/33] Update compile --- bin/compile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index ef19fcebb..6c0ddbbb1 100755 --- a/bin/compile +++ b/bin/compile @@ -85,7 +85,7 @@ export LANG=en_US.UTF-8 export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib -export LD_LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config # Switch to the repo's context. @@ -109,13 +109,14 @@ if [ ! -f runtime.txt ]; then fi # ### The Cache +puts-step "Creating cache directory in $CACHE_DIR" mkdir -p $CACHE_DIR # Purge "old-style" virtualenvs. -bpwatch start clear_old_venvs - [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include - [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src -bpwatch stop clear_old_venvs +#bpwatch start clear_old_venvs +# [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include +# [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src +#bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache @@ -238,6 +239,8 @@ source $BIN_DIR/steps/scikit-learn # ### Finalize # +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib + # Set context environment variables. set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true From 711add2f6516f72bbc35a75a9f122ff1f0844432 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 14:43:51 -0500 Subject: [PATCH 14/33] Update compile --- bin/compile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/compile b/bin/compile index 6c0ddbbb1..21aeb597a 100755 --- a/bin/compile +++ b/bin/compile @@ -111,6 +111,7 @@ fi # ### The Cache puts-step "Creating cache directory in $CACHE_DIR" mkdir -p $CACHE_DIR +ls -l $CACHE_DIR # Purge "old-style" virtualenvs. #bpwatch start clear_old_venvs @@ -239,8 +240,11 @@ source $BIN_DIR/steps/scikit-learn # ### Finalize # +export LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib +puts-step "Setting environment variables (LD_LIBRARY_PATH = $LD_LIBRARY_PATH)" + # Set context environment variables. set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true From d96345b7bed5c12f8fb370cbb17ab4b8fb27193b Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 3 Dec 2013 21:59:59 +0000 Subject: [PATCH 15/33] Remove collectstatic (ran separately) --- bin/compile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 6aefdb280..69e634ce5 100755 --- a/bin/compile +++ b/bin/compile @@ -170,8 +170,9 @@ source $BIN_DIR/steps/cryptography sub-env $BIN_DIR/steps/pip-install # Django collectstatic support. -sub-env $BIN_DIR/steps/collectstatic - +#bpwatch start collectstatic +# source $BIN_DIR/steps/collectstatic +#bpwatch stop collectstatic # ### Finalize # From c05a7c8422a20d56215ed8e0317824406c3c52fb Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:13:03 -0600 Subject: [PATCH 16/33] Create numpy-scipy https://github.com/dbrgn/heroku-buildpack-python-sklearn --- bin/steps/numpy-scipy | 69 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 bin/steps/numpy-scipy diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy new file mode 100644 index 000000000..a5bf7d5fb --- /dev/null +++ b/bin/steps/numpy-scipy @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# This script serves as the Scipy/Numpy build step of the +# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) +# compiler. +# +# A [buildpack](http://devcenter.heroku.com/articles/buildpacks) is an +# adapter between a Python application and Heroku's runtime. +# +# This script is invoked by [`bin/compile`](/). + +VENDORED_GIT_NAME="npscipy-binaries" +VENDORED_GIT_REPO="git://github.com/wyn/${VENDORED_GIT_NAME}.git" +# The location of the pre-compiled atlas/lapack/blas/gfortran binaries. +VENDORED_BINARIES="npscipy" +# the location of the bdist of numpy +VENDORED_NUMPY="numpy-1.7.0" +# the location of the bdist of scipy +VENDORED_SCIPY="scipy-0.11.0" + +# Syntax sugar. +source $BIN_DIR/utils + +# If numpy or scipy exists within requirements.txt then use binaries. +if (grep -iq -e "numpy" -e "scipy" requirements.txt) then + puts-step "Noticed numpy/scipy. Bootstrapping prebuilt binaries." + cd .heroku + + ## check whether this has happened already + ## set BLAS/LAPACK/ATLAS + if [ -d "vendor/lib/atlas-base" ]; then + puts-step "Using cached binaries." + else + puts-step "Creating/downloading binaries." + # Download and extract everything into target vendor directory. + git clone ${VENDORED_GIT_REPO} > /dev/null + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_BINARIES}.tar.gz > /dev/null + fi + export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a + export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a + export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a + export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + + if [ -d "python/lib/python2.7/site-packages/numpy" ]; then + puts-step "Using cached numpy." + else + puts-step "Creating/downloading numpy bdist." + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_NUMPY}.tar.gz > /dev/null + fi + + if [ -d "python/lib/python2.7/site-packages/scipy" ]; then + puts-step "Using cached scipy." + else + puts-step "Creating/downloading scipy bdist." + tar -xvf ${VENDORED_GIT_NAME}/${VENDORED_SCIPY}.tar.gz > /dev/null + fi + + # Move everything from venv directory to python directory + if [ -d "venv" ]; then + rsync -q -a venv/ python/ + rm -r venv + fi + + if [ -d ${VENDORED_GIT_NAME} ]; then + rm -rf ${VENDORED_GIT_NAME} + fi + cd .. +fi From 92b569fa526dc57044f2e213a0a3681b87bf9baa Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:14:33 -0600 Subject: [PATCH 17/33] Create scikit-learn From https://github.com/dbrgn/heroku-buildpack-python-sklearn --- bin/steps/scikit-learn | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 bin/steps/scikit-learn diff --git a/bin/steps/scikit-learn b/bin/steps/scikit-learn new file mode 100644 index 000000000..c19faa5ee --- /dev/null +++ b/bin/steps/scikit-learn @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# This script serves as the Skikit-learn build step of the +# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) +# compiler. +# +# A [buildpack](http://devcenter.heroku.com/articles/buildpacks) is an +# adapter between a Python application and Heroku's runtime. +# +# This script is invoked by [`bin/compile`](/). + +# Syntax sugar. +source $BIN_DIR/utils + +# If skikit-learn exists within requirements.txt then use binaries. +if (grep -iq -e "^\s*scikit-learn" requirements.txt) then + TARGET_SKLEARN_VERSION=$(cat requirements.txt | grep scikit-learn | tr -d " " | cut -c 13-) + puts-step "Noticed scikit-learn. Installing..." + cd .heroku + + ## check whether this has happened already + + if [ -d "python/lib/python2.7/site-packages/scikit-learn*" ]; then + puts-step "Using cached scikit-learn." + else + puts-step "Installing scikit-learn$TARGET_SKLEARN_VERSION via pip, but setting environment vars first." + export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a + export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a + export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a + export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + + pip install --use-mirrors scikit-learn$TARGET_SKLEARN_VERSION + fi + + cd .. + +fi From 5cdff5614f7986704c3b81aca704f35c967bd9ff Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:15:59 -0600 Subject: [PATCH 18/33] Add numpy and scikit-learn From https://github.com/dbrgn/heroku-buildpack-python-sklearn/commit/87cf7b24a358b916deaf26b784ea95be42590efe --- bin/compile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index 69e634ce5..7bbdb06bb 100755 --- a/bin/compile +++ b/bin/compile @@ -166,14 +166,15 @@ source $BIN_DIR/steps/pylibmc # Libffi support. source $BIN_DIR/steps/cryptography +# Numpy/scipy support +source $BIN_DIR/steps/numpy-scipy + +# Force scikit-learn to use a specific script +source $BIN_DIR/steps/scikit-learn + # Install dependencies with Pip. sub-env $BIN_DIR/steps/pip-install -# Django collectstatic support. -#bpwatch start collectstatic -# source $BIN_DIR/steps/collectstatic -#bpwatch stop collectstatic - # ### Finalize # From 81df154296d4657aff24c8a342ccab2af145981b Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:29:03 -0600 Subject: [PATCH 19/33] Replaced rsync with cp --- bin/steps/numpy-scipy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy index a5bf7d5fb..c8f530e8c 100644 --- a/bin/steps/numpy-scipy +++ b/bin/steps/numpy-scipy @@ -58,7 +58,7 @@ if (grep -iq -e "numpy" -e "scipy" requirements.txt) then # Move everything from venv directory to python directory if [ -d "venv" ]; then - rsync -q -a venv/ python/ + cp -a venv/* python/ rm -r venv fi From c4acbc5b75984009d9ea09e74289cd465262060e Mon Sep 17 00:00:00 2001 From: epervago Date: Mon, 23 Dec 2013 13:55:25 -0600 Subject: [PATCH 20/33] Update compile --- bin/compile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/compile b/bin/compile index 7bbdb06bb..5de5319a6 100755 --- a/bin/compile +++ b/bin/compile @@ -89,11 +89,19 @@ BUILD_DIR=$APP_DIR export PATH=$BUILD_DIR/.heroku/python/bin:$BUILD_DIR/.heroku/vendor/bin:$PATH export PYTHONUNBUFFERED=1 export LANG=en_US.UTF-8 +<<<<<<< HEAD export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include:/app/.heroku/python/include export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include:/app/.heroku/python/include export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config +======= +export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include +export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include +export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib +export LD_LIBRARY_PATH=$LIBRARY_PATH +export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config +>>>>>>> Update compile # Switch to the repo's context. cd $BUILD_DIR From 7e77c6bb4fa566f9082f6f4aff8c0ca977324aee Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:39:04 -0500 Subject: [PATCH 21/33] Add libffi step https://raw.githubusercontent.com/mfenniak/heroku-buildpack-python-libffi/master/bin/steps/libffi --- bin/steps/libffi | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 bin/steps/libffi diff --git a/bin/steps/libffi b/bin/steps/libffi new file mode 100644 index 000000000..9c11d055f --- /dev/null +++ b/bin/steps/libffi @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# This script is invoked by [`bin/compile`](/). + +# The location of the pre-compiled libffi binary. +VENDORED_LIBFFI="https://s3-us-west-2.amazonaws.com/mfenniak-graphviz/libffi-3.0.tgz" +VENDORED_PATH=/app/vendor/libffi-3.0 + +# Syntax sugar. +source $BIN_DIR/utils + +# Use vendored libffi. +echo "-----> Noticed cffi. Bootstrapping libffi." +PREV_DIR=$(pwd) +mkdir -p "$VENDORED_PATH" +cd "$VENDORED_PATH" + +# Download and extract libffi into target vendor directory. +curl -s -L -o tmp-libffi.tar.gz $VENDORED_LIBFFI +tar -zxvf tmp-libffi.tar.gz > /dev/null +rm tmp-libffi.tar.gz + +# Add libffi's to PKG_CONFIG_PATH +export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/lib/pkgconfig/ +echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" +if [ -e $(pwd)/lib/pkgconfig/libffi.pc ] +then + echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" +fi + +cd "$PREV_DIR" + + + From e0131df05b390d397d4c05e68aafd5ffebf61077 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:41:06 -0500 Subject: [PATCH 22/33] Update compile --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 5de5319a6..b2231b877 100755 --- a/bin/compile +++ b/bin/compile @@ -99,7 +99,7 @@ export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/ven export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib -export LD_LIBRARY_PATH=$LIBRARY_PATH +export LD_LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config >>>>>>> Update compile From 12bed04b8fe22c4f847608a74748ee7a6c5637ed Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:47:48 -0500 Subject: [PATCH 23/33] Update libffi --- bin/steps/libffi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/steps/libffi b/bin/steps/libffi index 9c11d055f..39a2ad0a0 100644 --- a/bin/steps/libffi +++ b/bin/steps/libffi @@ -28,6 +28,8 @@ then echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" fi +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ + cd "$PREV_DIR" From abb190ac94e156d46df59aad344e411b2e828758 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:49:18 -0500 Subject: [PATCH 24/33] Update libffi --- bin/steps/libffi | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/steps/libffi b/bin/steps/libffi index 39a2ad0a0..aec08bf73 100644 --- a/bin/steps/libffi +++ b/bin/steps/libffi @@ -3,7 +3,7 @@ # This script is invoked by [`bin/compile`](/). # The location of the pre-compiled libffi binary. -VENDORED_LIBFFI="https://s3-us-west-2.amazonaws.com/mfenniak-graphviz/libffi-3.0.tgz" +VENDORED_LIBFFI="https://s3.amazonaws.com/rt-uploads/libffi-3.0.tgz" VENDORED_PATH=/app/vendor/libffi-3.0 # Syntax sugar. @@ -22,15 +22,11 @@ rm tmp-libffi.tar.gz # Add libffi's to PKG_CONFIG_PATH export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/lib/pkgconfig/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" if [ -e $(pwd)/lib/pkgconfig/libffi.pc ] then echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" fi -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ - cd "$PREV_DIR" - - - From f62e0a641712cffe20e5baa37edbb8172b30fdd8 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:55:32 -0500 Subject: [PATCH 25/33] Don --- bin/steps/numpy-scipy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/steps/numpy-scipy b/bin/steps/numpy-scipy index c8f530e8c..3d5434d7a 100644 --- a/bin/steps/numpy-scipy +++ b/bin/steps/numpy-scipy @@ -39,8 +39,8 @@ if (grep -iq -e "numpy" -e "scipy" requirements.txt) then export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a - export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas - export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LIBRARY_PATH=$LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas if [ -d "python/lib/python2.7/site-packages/numpy" ]; then puts-step "Using cached numpy." From 737c2eee0d88bbc965844c4baff037c7a12f52f3 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 13:56:31 -0500 Subject: [PATCH 26/33] Don't overwrite *_LIBRARY_PATH --- bin/steps/scikit-learn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/steps/scikit-learn b/bin/steps/scikit-learn index c19faa5ee..c6ec9b125 100644 --- a/bin/steps/scikit-learn +++ b/bin/steps/scikit-learn @@ -27,8 +27,8 @@ if (grep -iq -e "^\s*scikit-learn" requirements.txt) then export BLAS=$(pwd)/vendor/lib/atlas-base/atlas/libblas.a export LAPACK=$(pwd)/vendor/lib/atlas-base/atlas/liblapack.a export ATLAS=$(pwd)/vendor/lib/atlas-base/libatlas.a - export LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas - export LD_LIBRARY_PATH=$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas + export LIBRARY_PATH=$LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/lib/atlas-base:$(pwd)/lib/atlas-base/atlas + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/vendor/lib:$(pwd)/vendor/lib/atlas-base:$(pwd)/vendor/lib/atlas-base/atlas pip install --use-mirrors scikit-learn$TARGET_SKLEARN_VERSION fi From 41634baae9c786f90b3a85f36b26871fb6158160 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 14:25:55 -0500 Subject: [PATCH 27/33] Update compile --- bin/compile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index b2231b877..73e8fd304 100755 --- a/bin/compile +++ b/bin/compile @@ -99,7 +99,7 @@ export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/ven export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib -export LD_LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config >>>>>>> Update compile @@ -134,13 +134,14 @@ if [ ! -f runtime.txt ]; then fi # ### The Cache +puts-step "Creating cache directory in $CACHE_DIR" mkdir -p $CACHE_DIR # Purge "old-style" virtualenvs. -bpwatch start clear_old_venvs - [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include - [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src -bpwatch stop clear_old_venvs +#bpwatch start clear_old_venvs +# [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include +# [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src +#bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache @@ -186,6 +187,8 @@ sub-env $BIN_DIR/steps/pip-install # ### Finalize # +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib + # Set context environment variables. set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true From 7e710626137f4039275ae147b15a21031be29829 Mon Sep 17 00:00:00 2001 From: epervago Date: Thu, 3 Jul 2014 14:43:51 -0500 Subject: [PATCH 28/33] Update compile --- bin/compile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/compile b/bin/compile index 73e8fd304..dabaa2cdd 100755 --- a/bin/compile +++ b/bin/compile @@ -136,6 +136,7 @@ fi # ### The Cache puts-step "Creating cache directory in $CACHE_DIR" mkdir -p $CACHE_DIR +ls -l $CACHE_DIR # Purge "old-style" virtualenvs. #bpwatch start clear_old_venvs @@ -187,8 +188,11 @@ sub-env $BIN_DIR/steps/pip-install # ### Finalize # +export LIBRARY_PATH=$LIBRARY_PATH:/app/vendor/libffi-3.0/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/vendor/libffi-3.0/lib +puts-step "Setting environment variables (LD_LIBRARY_PATH = $LD_LIBRARY_PATH)" + # Set context environment variables. set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true From b59f574818f681bfd2255eabbc5774b775dee80c Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 10 Feb 2015 10:37:29 -0600 Subject: [PATCH 29/33] Remove redundant libffi step (now taken care of by cryptography step) --- bin/steps/libffi | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 bin/steps/libffi diff --git a/bin/steps/libffi b/bin/steps/libffi deleted file mode 100644 index aec08bf73..000000000 --- a/bin/steps/libffi +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# This script is invoked by [`bin/compile`](/). - -# The location of the pre-compiled libffi binary. -VENDORED_LIBFFI="https://s3.amazonaws.com/rt-uploads/libffi-3.0.tgz" -VENDORED_PATH=/app/vendor/libffi-3.0 - -# Syntax sugar. -source $BIN_DIR/utils - -# Use vendored libffi. -echo "-----> Noticed cffi. Bootstrapping libffi." -PREV_DIR=$(pwd) -mkdir -p "$VENDORED_PATH" -cd "$VENDORED_PATH" - -# Download and extract libffi into target vendor directory. -curl -s -L -o tmp-libffi.tar.gz $VENDORED_LIBFFI -tar -zxvf tmp-libffi.tar.gz > /dev/null -rm tmp-libffi.tar.gz - -# Add libffi's to PKG_CONFIG_PATH -export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/lib/pkgconfig/ -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/lib/ -echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" -if [ -e $(pwd)/lib/pkgconfig/libffi.pc ] -then - echo "libffi.pc is in-place at $(pwd)/vendor/lib/pkgconfig/libffi.pc, libffi-based builds should work!" -fi - -cd "$PREV_DIR" From f52eb18897fb674222b667286821a99530e0fa01 Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 10 Feb 2015 14:01:29 -0600 Subject: [PATCH 30/33] Don't sanitize LD_LIBRARY_PATH/LIBRARY_PATH --- bin/compile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/compile b/bin/compile index 2b5473b23..3e1da2b05 100755 --- a/bin/compile +++ b/bin/compile @@ -46,8 +46,8 @@ export PATH=$PATH:$ROOT_DIR/vendor/pip-pop [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK -# Sanitizing environment variables. -unset GIT_DIR PYTHONHOME PYTHONPATH LD_LIBRARY_PATH LIBRARY_PATH +# Sanitizing environment variables.(don't reset LD_LIBRARY_PATH/LIBRARY_PATH after apt buildpack) +unset GIT_DIR PYTHONHOME PYTHONPATH # LD_LIBRARY_PATH LIBRARY_PATH bpwatch init $LOGPLEX_KEY bpwatch build python $BUILDPACK_VERSION $REQUEST_ID @@ -189,8 +189,8 @@ puts-step "Setting environment variables (LD_LIBRARY_PATH = $LD_LIBRARY_PATH)" set-env PATH '$HOME/.heroku/python/bin:$PATH' set-env PYTHONUNBUFFERED true set-env PYTHONHOME /app/.heroku/python -set-env LIBRARY_PATH /app/.heroku/vendor/lib:/app/.heroku/python/lib -set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH' +set-env LIBRARY_PATH /app/.heroku/vendor/lib:/app/.heroku/python/lib:/app/vendor/libffi-3.0/lib +set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:/app/vendor/libffi-3.0/lib:$LD_LIBRARY_PATH' set-default-env LANG en_US.UTF-8 set-default-env PYTHONHASHSEED random set-default-env PYTHONPATH /app/ From 8c08f85ec42158929d14b288c28c23ee60c34e85 Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 10 Feb 2015 14:06:15 -0600 Subject: [PATCH 31/33] Another attempt to play nicely with apt --- bin/compile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 3e1da2b05..a7c22fb11 100755 --- a/bin/compile +++ b/bin/compile @@ -46,8 +46,17 @@ export PATH=$PATH:$ROOT_DIR/vendor/pip-pop [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK -# Sanitizing environment variables.(don't reset LD_LIBRARY_PATH/LIBRARY_PATH after apt buildpack) -unset GIT_DIR PYTHONHOME PYTHONPATH # LD_LIBRARY_PATH LIBRARY_PATH +# Sanitizing environment variables. +unset GIT_DIR PYTHONHOME PYTHONPATH LD_LIBRARY_PATH LIBRARY_PATH + +# Add apt-related paths +export PATH="$BUILD_DIR/.apt/usr/bin:$PATH" +export LD_LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LD_LIBRARY_PATH" +export LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LIBRARY_PATH" +export INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$INCLUDE_PATH" +export CPATH="$INCLUDE_PATH" +export CPPPATH="$INCLUDE_PATH" +export PKG_CONFIG_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/pkgconfig:$PKG_CONFIG_PATH" bpwatch init $LOGPLEX_KEY bpwatch build python $BUILDPACK_VERSION $REQUEST_ID From efad5aab776e33fc5b5d7e3cfb97a57a1570e4f8 Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 10 Feb 2015 14:11:38 -0600 Subject: [PATCH 32/33] Attempt to make it play nicely with apt v2 --- bin/compile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bin/compile b/bin/compile index a7c22fb11..5a41fef39 100755 --- a/bin/compile +++ b/bin/compile @@ -49,15 +49,6 @@ export PATH=$PATH:$ROOT_DIR/vendor/pip-pop # Sanitizing environment variables. unset GIT_DIR PYTHONHOME PYTHONPATH LD_LIBRARY_PATH LIBRARY_PATH -# Add apt-related paths -export PATH="$BUILD_DIR/.apt/usr/bin:$PATH" -export LD_LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LD_LIBRARY_PATH" -export LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LIBRARY_PATH" -export INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$INCLUDE_PATH" -export CPATH="$INCLUDE_PATH" -export CPPPATH="$INCLUDE_PATH" -export PKG_CONFIG_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/pkgconfig:$PKG_CONFIG_PATH" - bpwatch init $LOGPLEX_KEY bpwatch build python $BUILDPACK_VERSION $REQUEST_ID TMP_APP_DIR=$CACHE_DIR/tmp_app_dir @@ -104,6 +95,14 @@ export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/. export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config +# Add apt-related paths +export LD_LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LD_LIBRARY_PATH" +export LIBRARY_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu:$BUILD_DIR/.apt/usr/lib:$LIBRARY_PATH" +export INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$INCLUDE_PATH" +export C_INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$C_INCLUDE_PATH" +export CPLUS_INCLUDE_PATH="$BUILD_DIR/.apt/usr/include:$CPLUS_INCLUDE_PATH" +export PKG_CONFIG_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/i386-linux-gnu/pkgconfig:$BUILD_DIR/.apt/usr/lib/pkgconfig:$PKG_CONFIG_PATH" + # Switch to the repo's context. cd $BUILD_DIR From 1f33489f2afbbfe611c74b06f6e6a99519104a36 Mon Sep 17 00:00:00 2001 From: Evgeny Pervago Date: Tue, 10 Feb 2015 14:22:23 -0600 Subject: [PATCH 33/33] Temp debug statements --- bin/steps/pip-install | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 05b97ab78..356179f4c 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -5,6 +5,9 @@ source $BIN_DIR/utils # Install dependencies with Pip. puts-step "Installing dependencies with pip" +echo $LIBRARY_PATH +echo $INCLUDE_PATH + [ ! "$FRESH_PYTHON" ] && bpwatch start pip_install [ "$FRESH_PYTHON" ] && bpwatch start pip_install_first