From a33519ce452245f915648867b97f460715c7ab5b Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Fri, 31 Jan 2020 15:44:56 +0530 Subject: [PATCH 01/31] Use cmake3. --- Dockerfile | 3 ++- build_wheels_linux.sh | 2 +- {DockerHub => hub.docker.com}/Dockerfile | 0 {DockerHub => hub.docker.com}/Dockerfile.moose | 0 {DockerHub => hub.docker.com}/Makefile | 0 {DockerHub => hub.docker.com}/build_docker.sh | 0 {DockerHub => hub.docker.com}/launch_docker.sh | 0 7 files changed, 3 insertions(+), 2 deletions(-) rename {DockerHub => hub.docker.com}/Dockerfile (100%) rename {DockerHub => hub.docker.com}/Dockerfile.moose (100%) rename {DockerHub => hub.docker.com}/Makefile (100%) rename {DockerHub => hub.docker.com}/build_docker.sh (100%) rename {DockerHub => hub.docker.com}/launch_docker.sh (100%) diff --git a/Dockerfile b/Dockerfile index 00b2d24..c175b70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM quay.io/pypa/manylinux2010_x86_64 +FROM quay.io/pypa/manylinux2014_x86_64 +MAINTAINER Dilawar Singh ARG PYPI_PASSWORD diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index efbed40..ac88013 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -41,7 +41,7 @@ fi # Try to link statically. GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" -CMAKE=/usr/bin/cmake28 +CMAKE=/usr/bin/cmake3 # Build wheels here. for PYV in 36 27; do diff --git a/DockerHub/Dockerfile b/hub.docker.com/Dockerfile similarity index 100% rename from DockerHub/Dockerfile rename to hub.docker.com/Dockerfile diff --git a/DockerHub/Dockerfile.moose b/hub.docker.com/Dockerfile.moose similarity index 100% rename from DockerHub/Dockerfile.moose rename to hub.docker.com/Dockerfile.moose diff --git a/DockerHub/Makefile b/hub.docker.com/Makefile similarity index 100% rename from DockerHub/Makefile rename to hub.docker.com/Makefile diff --git a/DockerHub/build_docker.sh b/hub.docker.com/build_docker.sh similarity index 100% rename from DockerHub/build_docker.sh rename to hub.docker.com/build_docker.sh diff --git a/DockerHub/launch_docker.sh b/hub.docker.com/launch_docker.sh similarity index 100% rename from DockerHub/launch_docker.sh rename to hub.docker.com/launch_docker.sh From d54db91c71f2cb1796e677f9cff6731f09b0ab67 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Fri, 31 Jan 2020 16:12:15 +0530 Subject: [PATCH 02/31] Making things bit modular. --- BuildImages/Dockerfile | 13 +++++ BuildImages/Makefile | 7 +++ BuildImages/build_wheels_linux.sh | 95 +++++++++++++++++++++++++++++++ Dockerfile | 14 ++--- 4 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 BuildImages/Dockerfile create mode 100644 BuildImages/Makefile create mode 100755 BuildImages/build_wheels_linux.sh diff --git a/BuildImages/Dockerfile b/BuildImages/Dockerfile new file mode 100644 index 0000000..e55969e --- /dev/null +++ b/BuildImages/Dockerfile @@ -0,0 +1,13 @@ +FROM quay.io/pypa/manylinux2014_x86_64 +MAINTAINER Dilawar Singh + +ARG PYPI_PASSWORD + +MAINTAINER Dilawar Singh +ENV PATH=/usr/local/bin:$PATH +RUN yum install -y cmake3 wget vim git +COPY ./build_wheels_linux.sh /opt/build_wheels_linunx.sh +RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz && \ + tar xvf gsl*.tar.gz && \ + cd gsl* && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS && make install +RUN git clone https://github.com/dilawar/moose-core --depth 100 diff --git a/BuildImages/Makefile b/BuildImages/Makefile new file mode 100644 index 0000000..3a4bc6e --- /dev/null +++ b/BuildImages/Makefile @@ -0,0 +1,7 @@ +IMAGE := bhallalab/manylinux + +all : Dockerfile + docker build -t $(IMAGE) . + +run : + docker run -ilt $(IMAGE) bash diff --git a/BuildImages/build_wheels_linux.sh b/BuildImages/build_wheels_linux.sh new file mode 100755 index 0000000..ac88013 --- /dev/null +++ b/BuildImages/build_wheels_linux.sh @@ -0,0 +1,95 @@ +#!/bin/sh + +set -e +set -x + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) +NUM_WORKERS=$((NPROC/2)) + +if [ "$TRAVIS" == "true" ]; then + NUM_WORKERS=2 +fi +MAKEOPTS="-j$NUM_WORKERS" + +# Place to store wheels. +WHEELHOUSE=${1-$HOME/wheelhouse} +echo "Path to store wheels : $WHEELHOUSE" +mkdir -p $WHEELHOUSE + +# tag on github and revision number. Make sure that they are there. +BRANCH=$(cat ./BRANCH) +VERSION="3.2.0.dev$(date +%Y%m%d)" + +echo "Building version $REVISION, from branch $BRANCH" + +if [ ! -f /usr/local/lib/libgsl.a ]; then + #wget --no-check-certificate ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz + curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz + tar xvf gsl-2.4.tar.gz + cd gsl-2.4 + CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS + make install + cd .. +fi + +MOOSE_SOURCE_DIR=$SCRIPT_DIR/moose-core + +if [ ! -d $MOOSE_SOURCE_DIR ]; then + git clone https://github.com/BhallaLab/moose-core --depth 10 --branch $BRANCH +fi + +# Try to link statically. +GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" +CMAKE=/usr/bin/cmake3 + +# Build wheels here. +for PYV in 36 27; do + PYDIR=/opt/python/cp${PYV}-cp${PYV}m + PYVER=$(basename $PYDIR) + mkdir -p $PYVER + ( + cd $PYVER + echo "Building using $PYDIR in $PYVER" + PYTHON=$(ls $PYDIR/bin/python?.?) + if [ "$PYV" -eq 27 ]; then + $PYTHON -m pip install numpy==1.15 + $PYTHON -m pip install matplotlib==2.2.3 + else + $PYTHON -m pip install numpy twine + $PYTHON -m pip install matplotlib + fi + $PYTHON -m pip install twine + $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" + git pull || echo "Failed to pull $BRANCH" + $CMAKE -DPYTHON_EXECUTABLE=$PYTHON \ + -DGSL_STATIC_LIBRARIES=$GSL_STATIC_LIBS \ + -DVERSION_MOOSE=$VERSION \ + ${MOOSE_SOURCE_DIR} + make $MAKEOPTS + + # Now build bdist_wheel + cd python + cp setup.cmake.py setup.py + $PYTHON -m pip wheel . -w $WHEELHOUSE + echo "Content of WHEELHOUSE" + ls -lh $WHEELHOUSE/*.whl + ) +done + +# List all wheels. +ls -lh $WHEELHOUSE/*.whl + +# now check the wheels. +for whl in $WHEELHOUSE/pymoose*.whl; do + auditwheel show "$whl" +done + +echo "Installing before testing ... " +/opt/python/cp27-cp27m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl +/opt/python/cp36-cp36m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +for PYV in 36 27; do + PYDIR=/opt/python/cp${PYV}-cp${PYV}m + PYTHON=$(ls $PYDIR/bin/python?.?) + $PYTHON -c 'import moose; print( moose.__version__ )' +done diff --git a/Dockerfile b/Dockerfile index c175b70..a680575 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,12 +5,8 @@ ARG PYPI_PASSWORD MAINTAINER Dilawar Singh ENV PATH=/usr/local/bin:$PATH -RUN yum update -y -RUN yum install -y cmake3 -RUN yum install -y wget -RUN wget https://github.com/BhallaLab/deploy/archive/master.tar.gz -RUN ls -la *.gz -RUN tar xvf master.tar.gz -RUN cd deploy-master && ./build_wheels_linux.sh -RUN echo "pass $PYPI_PASSWORD" -RUN cd deploy-master && ./test_and_upload.sh "$PYPI_PASSWORD" +RUN yum install -y cmake3 wget vim +COPY ./build_wheels_linux.sh /opt/build_wheels_linunx.sh +RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz +RUN curl -O https://github.com/BhallaLab/deploy/archive/master.tar.gz +CMD bash From c19aa9b017f00a03809798b8ac62402d2e34462f Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Fri, 31 Jan 2020 17:01:05 +0530 Subject: [PATCH 03/31] Building with 2014. --- BuildImages2014/Dockerfile | 15 +++++ BuildImages2014/Makefile | 7 ++ BuildImages2014/build_wheels_linux.sh | 97 +++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 BuildImages2014/Dockerfile create mode 100644 BuildImages2014/Makefile create mode 100755 BuildImages2014/build_wheels_linux.sh diff --git a/BuildImages2014/Dockerfile b/BuildImages2014/Dockerfile new file mode 100644 index 0000000..8727fa4 --- /dev/null +++ b/BuildImages2014/Dockerfile @@ -0,0 +1,15 @@ +FROM quay.io/pypa/manylinux2014_x86_64 +MAINTAINER Dilawar Singh + +ARG PYPI_PASSWORD + +MAINTAINER Dilawar Singh +ENV PATH=/usr/local/bin:$PATH +RUN yum install -y cmake3 wget vim git +WORKDIR /root +RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz && \ + tar xvf gsl*.tar.gz && \ + cd gsl* && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS && make install +RUN git clone https://github.com/dilawar/moose-core --depth 100 +COPY . /root/deploy +CMD [ "bash" ] diff --git a/BuildImages2014/Makefile b/BuildImages2014/Makefile new file mode 100644 index 0000000..acb5f9b --- /dev/null +++ b/BuildImages2014/Makefile @@ -0,0 +1,7 @@ +IMAGE := bhallalab/manylinux + +all : Dockerfile + docker build -t $(IMAGE) . + +run : + docker run -it $(IMAGE) bash diff --git a/BuildImages2014/build_wheels_linux.sh b/BuildImages2014/build_wheels_linux.sh new file mode 100755 index 0000000..5957226 --- /dev/null +++ b/BuildImages2014/build_wheels_linux.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +set -e -x -u + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) +NUM_WORKERS=$((NPROC+1)) + +if [ "$TRAVIS" == "true" ]; then + NUM_WORKERS=3 +fi +MAKEOPTS="-j$NUM_WORKERS" + +# Place to store wheels. +WHEELHOUSE=${1-$HOME/wheelhouse} +echo "Path to store wheels : $WHEELHOUSE" +mkdir -p $WHEELHOUSE + +# tag on github and revision number. Make sure that they are there. +[ -f ./BRANCH ] || echo "master" >> ./BRANCH +BRANCH=$(cat ./BRANCH) +VERSION="3.2.0.dev$(date +%Y%m%d)" + +echo "Building version $REVISION, from branch $BRANCH" + +GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" + +if [ ! -f /usr/local/lib/libgsl.a ]; then + #wget --no-check-certificate ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz + curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz + tar xvf gsl-2.4.tar.gz + cd gsl-2.4 + CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS + make install + cd .. +fi + +MOOSE_SOURCE_DIR=$SCRIPT_DIR/moose-core + +if [ ! -d $MOOSE_SOURCE_DIR ]; then + git clone https://github.com/dilawar/moose-core --depth 10 --branch $BRANCH +fi + +# Try to link statically. +CMAKE=/usr/bin/cmake3 + +# Build wheels here. +for PYV in 38 36 27; do + PYDIR=/opt/python/cp${PYV}-cp${PYV}m + PYVER=$(basename $PYDIR) + mkdir -p $PYVER + ( + cd $PYVER + echo "Building using $PYDIR in $PYVER" + PYTHON=$(ls $PYDIR/bin/python?.?) + if [ "$PYV" -eq 27 ]; then + $PYTHON -m pip install numpy==1.15 + $PYTHON -m pip install matplotlib==2.2.3 + else + $PYTHON -m pip install numpy twine + $PYTHON -m pip install matplotlib + fi + $PYTHON -m pip install twine + $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" + git pull || echo "Failed to pull $BRANCH" + $CMAKE -DPYTHON_EXECUTABLE=$PYTHON \ + -DGSL_STATIC_LIBRARIES=$GSL_STATIC_LIBS \ + -DVERSION_MOOSE=$VERSION \ + ${MOOSE_SOURCE_DIR} + make $MAKEOPTS + + # Now build bdist_wheel + cd python + cp setup.cmake.py setup.py + $PYTHON -m pip wheel . -w $WHEELHOUSE + echo "Content of WHEELHOUSE" + ls -lh $WHEELHOUSE/*.whl + ) +done + +# List all wheels. +ls -lh $WHEELHOUSE/*.whl + +# now check the wheels. +for whl in $WHEELHOUSE/pymoose*.whl; do + auditwheel show "$whl" +done + +echo "Installing before testing ... " +/opt/python/cp27-cp27m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl +/opt/python/cp36-cp36m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +/opt/python/cp38-cp38m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +for PYV in 38 36 27; do + PYDIR=/opt/python/cp${PYV}-cp${PYV} + PYTHON=$(ls $PYDIR/bin/python?.?) + $PYTHON -c 'import moose; print(moose.__version__)' +done From d38b67d114d584fe48c5af5f0b8259a4cd903115 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Fri, 31 Jan 2020 17:13:26 +0530 Subject: [PATCH 04/31] Renamed. --- BuildImages/Dockerfile | 13 ---- BuildImages/Makefile | 7 -- BuildImages/build_wheels_linux.sh | 95 --------------------------- BuildImages2014/Dockerfile | 4 +- BuildImages2014/build_wheels_linux.sh | 24 +++---- 5 files changed, 12 insertions(+), 131 deletions(-) delete mode 100644 BuildImages/Dockerfile delete mode 100644 BuildImages/Makefile delete mode 100755 BuildImages/build_wheels_linux.sh diff --git a/BuildImages/Dockerfile b/BuildImages/Dockerfile deleted file mode 100644 index e55969e..0000000 --- a/BuildImages/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM quay.io/pypa/manylinux2014_x86_64 -MAINTAINER Dilawar Singh - -ARG PYPI_PASSWORD - -MAINTAINER Dilawar Singh -ENV PATH=/usr/local/bin:$PATH -RUN yum install -y cmake3 wget vim git -COPY ./build_wheels_linux.sh /opt/build_wheels_linunx.sh -RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz && \ - tar xvf gsl*.tar.gz && \ - cd gsl* && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS && make install -RUN git clone https://github.com/dilawar/moose-core --depth 100 diff --git a/BuildImages/Makefile b/BuildImages/Makefile deleted file mode 100644 index 3a4bc6e..0000000 --- a/BuildImages/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -IMAGE := bhallalab/manylinux - -all : Dockerfile - docker build -t $(IMAGE) . - -run : - docker run -ilt $(IMAGE) bash diff --git a/BuildImages/build_wheels_linux.sh b/BuildImages/build_wheels_linux.sh deleted file mode 100755 index ac88013..0000000 --- a/BuildImages/build_wheels_linux.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/sh - -set -e -set -x - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) -NUM_WORKERS=$((NPROC/2)) - -if [ "$TRAVIS" == "true" ]; then - NUM_WORKERS=2 -fi -MAKEOPTS="-j$NUM_WORKERS" - -# Place to store wheels. -WHEELHOUSE=${1-$HOME/wheelhouse} -echo "Path to store wheels : $WHEELHOUSE" -mkdir -p $WHEELHOUSE - -# tag on github and revision number. Make sure that they are there. -BRANCH=$(cat ./BRANCH) -VERSION="3.2.0.dev$(date +%Y%m%d)" - -echo "Building version $REVISION, from branch $BRANCH" - -if [ ! -f /usr/local/lib/libgsl.a ]; then - #wget --no-check-certificate ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz - curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz - tar xvf gsl-2.4.tar.gz - cd gsl-2.4 - CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS - make install - cd .. -fi - -MOOSE_SOURCE_DIR=$SCRIPT_DIR/moose-core - -if [ ! -d $MOOSE_SOURCE_DIR ]; then - git clone https://github.com/BhallaLab/moose-core --depth 10 --branch $BRANCH -fi - -# Try to link statically. -GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" -CMAKE=/usr/bin/cmake3 - -# Build wheels here. -for PYV in 36 27; do - PYDIR=/opt/python/cp${PYV}-cp${PYV}m - PYVER=$(basename $PYDIR) - mkdir -p $PYVER - ( - cd $PYVER - echo "Building using $PYDIR in $PYVER" - PYTHON=$(ls $PYDIR/bin/python?.?) - if [ "$PYV" -eq 27 ]; then - $PYTHON -m pip install numpy==1.15 - $PYTHON -m pip install matplotlib==2.2.3 - else - $PYTHON -m pip install numpy twine - $PYTHON -m pip install matplotlib - fi - $PYTHON -m pip install twine - $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" - git pull || echo "Failed to pull $BRANCH" - $CMAKE -DPYTHON_EXECUTABLE=$PYTHON \ - -DGSL_STATIC_LIBRARIES=$GSL_STATIC_LIBS \ - -DVERSION_MOOSE=$VERSION \ - ${MOOSE_SOURCE_DIR} - make $MAKEOPTS - - # Now build bdist_wheel - cd python - cp setup.cmake.py setup.py - $PYTHON -m pip wheel . -w $WHEELHOUSE - echo "Content of WHEELHOUSE" - ls -lh $WHEELHOUSE/*.whl - ) -done - -# List all wheels. -ls -lh $WHEELHOUSE/*.whl - -# now check the wheels. -for whl in $WHEELHOUSE/pymoose*.whl; do - auditwheel show "$whl" -done - -echo "Installing before testing ... " -/opt/python/cp27-cp27m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl -/opt/python/cp36-cp36m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl -for PYV in 36 27; do - PYDIR=/opt/python/cp${PYV}-cp${PYV}m - PYTHON=$(ls $PYDIR/bin/python?.?) - $PYTHON -c 'import moose; print( moose.__version__ )' -done diff --git a/BuildImages2014/Dockerfile b/BuildImages2014/Dockerfile index 8727fa4..8d74e79 100644 --- a/BuildImages2014/Dockerfile +++ b/BuildImages2014/Dockerfile @@ -10,6 +10,6 @@ WORKDIR /root RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz && \ tar xvf gsl*.tar.gz && \ cd gsl* && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS && make install -RUN git clone https://github.com/dilawar/moose-core --depth 100 -COPY . /root/deploy +COPY build_wheels_linux.sh . +# RUN ./build_wheels_linux.sh CMD [ "bash" ] diff --git a/BuildImages2014/build_wheels_linux.sh b/BuildImages2014/build_wheels_linux.sh index 5957226..8982937 100755 --- a/BuildImages2014/build_wheels_linux.sh +++ b/BuildImages2014/build_wheels_linux.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e -x -u +set -e -x SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) @@ -45,21 +45,16 @@ fi CMAKE=/usr/bin/cmake3 # Build wheels here. -for PYV in 38 36 27; do - PYDIR=/opt/python/cp${PYV}-cp${PYV}m +for PYV in 38 37m 36m; do + PYDIR=/opt/python/cp${PYV}-cp${PYV} PYVER=$(basename $PYDIR) mkdir -p $PYVER ( cd $PYVER echo "Building using $PYDIR in $PYVER" PYTHON=$(ls $PYDIR/bin/python?.?) - if [ "$PYV" -eq 27 ]; then - $PYTHON -m pip install numpy==1.15 - $PYTHON -m pip install matplotlib==2.2.3 - else - $PYTHON -m pip install numpy twine - $PYTHON -m pip install matplotlib - fi + $PYTHON -m pip install numpy twine + $PYTHON -m pip install matplotlib $PYTHON -m pip install twine $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" git pull || echo "Failed to pull $BRANCH" @@ -87,10 +82,11 @@ for whl in $WHEELHOUSE/pymoose*.whl; do done echo "Installing before testing ... " -/opt/python/cp27-cp27m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl -/opt/python/cp36-cp36m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl -/opt/python/cp38-cp38m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl -for PYV in 38 36 27; do + +/opt/python/cp36-cp36/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +/opt/python/cp38-cp38/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +for PYV in 38 36; do + /opt/python/cp${PYV}-cp${PYV}/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl PYDIR=/opt/python/cp${PYV}-cp${PYV} PYTHON=$(ls $PYDIR/bin/python?.?) $PYTHON -c 'import moose; print(moose.__version__)' From b3f0ac3768308e0557ead8bfb90cff5bfe8a5a65 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Fri, 31 Jan 2020 17:38:47 +0530 Subject: [PATCH 05/31] Minor update to Dockerfile. --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index a680575..5da1528 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM quay.io/pypa/manylinux2014_x86_64 +FROM quay.io/pypa/manylinux2010_x86_64 MAINTAINER Dilawar Singh - ARG PYPI_PASSWORD - -MAINTAINER Dilawar Singh -ENV PATH=/usr/local/bin:$PATH -RUN yum install -y cmake3 wget vim -COPY ./build_wheels_linux.sh /opt/build_wheels_linunx.sh -RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz -RUN curl -O https://github.com/BhallaLab/deploy/archive/master.tar.gz +WORKDIR /root +RUN yum install -y cmake3 git +RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz \ + && tar xvf gsl-2.4.tar.gz \ + && cd gsl-2.4 \ + && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS \ + && make install +COPY ./build_wheels_linux.sh . CMD bash From d4add5a7035709379bccbe03a6b4d3c052a611d8 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Fri, 31 Jan 2020 20:59:57 +0530 Subject: [PATCH 06/31] Docker build is successful. --- Dockerfile | 13 ++++++-- Makefile | 13 +++++--- build_wheels_linux.sh | 77 ++++++++++++++++++++----------------------- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5da1528..7b3699d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,21 @@ FROM quay.io/pypa/manylinux2010_x86_64 MAINTAINER Dilawar Singh + ARG PYPI_PASSWORD + +RUN yum install -y cmake3 git tree && rm -rf /var/cache/yum/* + +ENV PATH /usr/local/bin:$PATH +RUN ln -s /usr/bin/cmake3 /usr/local/bin/cmake + WORKDIR /root -RUN yum install -y cmake3 git + RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz \ && tar xvf gsl-2.4.tar.gz \ && cd gsl-2.4 \ && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS \ && make install + COPY ./build_wheels_linux.sh . -CMD bash +RUN ./build_wheels_linux.sh +CMD [ "bash", "-c", "./build_wheels_linux.sh"] diff --git a/Makefile b/Makefile index 872d090..84d5e43 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,14 @@ -VERSION:=$(shell cat ./VERSION) - all : wheels -DOCKERFILE:="bhallalab/pymoose_wheels_$(USER):$(VERSION)" +DOCKERFILE:="bhallalab/manylinux-moose:latest" wheels : ./Dockerfile ./build_wheels_linux.sh mkdir -p $(HOME)/wheelhouse - docker build --no-cache \ - -t $(DOCKERFILE) \ + docker build -t $(DOCKERFILE) \ --build-arg PYPI_PASSWORD=$(PYPI_PASSWORD) . + +run : ./Dockerfile + docker run -it $(DOCKERFILE) bash + +upload: + docker push $(DOCKERFILE) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index ac88013..c5a89bd 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -5,7 +5,7 @@ set -x SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) -NUM_WORKERS=$((NPROC/2)) +NUM_WORKERS=$((NPROC-1)) if [ "$TRAVIS" == "true" ]; then NUM_WORKERS=2 @@ -18,13 +18,13 @@ echo "Path to store wheels : $WHEELHOUSE" mkdir -p $WHEELHOUSE # tag on github and revision number. Make sure that they are there. +[ -f ./BRANCH ] || echo "master" > ./BRANCH BRANCH=$(cat ./BRANCH) VERSION="3.2.0.dev$(date +%Y%m%d)" echo "Building version $REVISION, from branch $BRANCH" if [ ! -f /usr/local/lib/libgsl.a ]; then - #wget --no-check-certificate ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz tar xvf gsl-2.4.tar.gz cd gsl-2.4 @@ -36,47 +36,41 @@ fi MOOSE_SOURCE_DIR=$SCRIPT_DIR/moose-core if [ ! -d $MOOSE_SOURCE_DIR ]; then - git clone https://github.com/BhallaLab/moose-core --depth 10 --branch $BRANCH + git clone https://github.com/dilawar/moose-core --depth 10 --branch $BRANCH fi -# Try to link statically. +# GSL will be linked statically. GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" -CMAKE=/usr/bin/cmake3 + +PY2=/opt/python/cp27-cp27m/bin/python2.7 +$PY2 -m pip install numpy==1.14 matplotlib==2.2.4 + +PY3=/opt/python/cp38-cp38/bin/python3.8 +$PY3 -m pip install numpy matplotlib # Build wheels here. -for PYV in 36 27; do - PYDIR=/opt/python/cp${PYV}-cp${PYV}m - PYVER=$(basename $PYDIR) - mkdir -p $PYVER - ( - cd $PYVER - echo "Building using $PYDIR in $PYVER" - PYTHON=$(ls $PYDIR/bin/python?.?) - if [ "$PYV" -eq 27 ]; then - $PYTHON -m pip install numpy==1.15 - $PYTHON -m pip install matplotlib==2.2.3 - else - $PYTHON -m pip install numpy twine - $PYTHON -m pip install matplotlib - fi - $PYTHON -m pip install twine - $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" - git pull || echo "Failed to pull $BRANCH" - $CMAKE -DPYTHON_EXECUTABLE=$PYTHON \ - -DGSL_STATIC_LIBRARIES=$GSL_STATIC_LIBS \ - -DVERSION_MOOSE=$VERSION \ - ${MOOSE_SOURCE_DIR} - make $MAKEOPTS - - # Now build bdist_wheel - cd python - cp setup.cmake.py setup.py - $PYTHON -m pip wheel . -w $WHEELHOUSE - echo "Content of WHEELHOUSE" - ls -lh $WHEELHOUSE/*.whl - ) +for PY in $PY3 $PY2; do + ( + BUILDIR=$(basename $PY) + mkdir -p $BUILDIR + cd $BUILDIR + echo "Building using in $PY" + git pull || echo "Failed to pull $BRANCH" + cmake -DPYTHON_EXECUTABLE=$PY \ + -DGSL_STATIC_LIBRARIES=$GSL_STATIC_LIBS \ + -DVERSION_MOOSE=$VERSION ${MOOSE_SOURCE_DIR} + make $MAKEOPTS + # Now build bdist_wheel + cd python + cp setup.cmake.py setup.py + $PY -m pip wheel . -w $WHEELHOUSE + echo "Content of WHEELHOUSE" + ls -lh $WHEELHOUSE/*.whl + ) done +$PY3 -m pip install twine auditwheel + # List all wheels. ls -lh $WHEELHOUSE/*.whl @@ -86,10 +80,9 @@ for whl in $WHEELHOUSE/pymoose*.whl; do done echo "Installing before testing ... " -/opt/python/cp27-cp27m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl -/opt/python/cp36-cp36m/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl -for PYV in 36 27; do - PYDIR=/opt/python/cp${PYV}-cp${PYV}m - PYTHON=$(ls $PYDIR/bin/python?.?) - $PYTHON -c 'import moose; print( moose.__version__ )' +$PY2 -m pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl +$PY3 -m pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl + +for PY in $PY3 $PY2; do + $PY -c 'import moose; print(moose.__version__)' done From 94baa4e8afcf8ec97c5c861339554fed2f71f5be Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 6 Apr 2020 18:47:05 +0530 Subject: [PATCH 07/31] Only python3 on OSX. --- build_wheels_osx.sh | 53 +++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/build_wheels_osx.sh b/build_wheels_osx.sh index e5c46a3..7849744 100755 --- a/build_wheels_osx.sh +++ b/build_wheels_osx.sh @@ -10,7 +10,6 @@ export PATH=/usr/local/bin:$PATH brew update || echo "Failed to update brew" brew install gsl || brew upgrade gsl brew install python@3 || echo "Failed to install python3" -brew install python@2 || echo "Failed to install python2" # Following are to remove numpy; It is breaking the build on Xcode9.4. # brew uninstall gdal postgis || echo "Failed to uninstall gdal/postgis" @@ -34,33 +33,31 @@ rm -rf $WHEELHOUSE && mkdir -p $WHEELHOUSE DELOCATE_WHEEL=/usr/local/bin/delocate-wheel # Always prefer brew version. -for _py in 3 2; do - PYTHON=/usr/local/bin/python$_py +PYTHON=/usr/local/bin/python3 - if [ ! -f $PYTHON ]; then - echo "Not found $PYTHON" - continue - fi - - - $PYTHON -m pip install setuptools --upgrade --user - $PYTHON -m pip install wheel --upgrade --user - $PYTHON -m pip install numpy --upgrade --user - $PYTHON -m pip install twine --upgrade --user +if [ ! -f $PYTHON ]; then + echo "Not found $PYTHON" + continue +fi - PLATFORM=$($PYTHON -c "import distutils.util; print(distutils.util.get_platform())") - ( - cd $MOOSE_SOURCE_DIR - $PYTHON setup.py build_ext - export GSL_USE_STATIC_LIBRARIES=1 - $PYTHON setup.py bdist_wheel --skip-build - $DELOCATE_WHEEL -v dist/*.whl -w $WHEELHOUSE - rm -rf dist/*.whl - ) - if [ ! -z "$PYMOOSE_PYPI_PASSWORD" ]; then - echo "Did you test the wheels? I am uploading anyway ..." - $PYTHON -m twine upload -u bhallalab -p $PYMOOSE_PYPI_PASSWORD \ - $WHEELHOUSE/pymoose*.whl || echo "Failed to upload to PyPi" - fi -done +$PYTHON -m pip install setuptools --upgrade --user +$PYTHON -m pip install wheel --upgrade --user +$PYTHON -m pip install numpy --upgrade --user +$PYTHON -m pip install twine --upgrade --user + +PLATFORM=$($PYTHON -c "import distutils.util; print(distutils.util.get_platform())") +( + cd $MOOSE_SOURCE_DIR + $PYTHON setup.py build_ext + export GSL_USE_STATIC_LIBRARIES=1 + $PYTHON setup.py bdist_wheel --skip-build + $DELOCATE_WHEEL -v dist/*.whl -w $WHEELHOUSE + rm -rf dist/*.whl +) + +if [ ! -z "$PYMOOSE_PYPI_PASSWORD" ]; then + echo "Did you test the wheels? I am uploading anyway ..." + $PYTHON -m twine upload -u bhallalab -p $PYMOOSE_PYPI_PASSWORD \ + $WHEELHOUSE/pymoose*.whl || echo "Failed to upload to PyPi" +fi From 49a579949bf2c73069b22fde0a2c931ac29811d9 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 03:03:17 +0530 Subject: [PATCH 08/31] pymoose deploy (forked) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fe83f0..6429520 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.org/BhallaLab/deploy.svg?branch=master)](https://travis-ci.org/BhallaLab/deploy) +[![Build +Status](https://travis-ci.org/dilawar/pymoose-deploy.svg?branch=master)](https://travis-ci.org/dilawar/pymoose-deploy) This repository contains all scripts required to build and release pymoose. From 2982136cae638344b236c6b42c05ce2c36c730f5 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 03:05:01 +0530 Subject: [PATCH 09/31] builds from BhallaLab/moose-core --- build_wheels_osx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_wheels_osx.sh b/build_wheels_osx.sh index 7849744..cb36f92 100755 --- a/build_wheels_osx.sh +++ b/build_wheels_osx.sh @@ -20,7 +20,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" MOOSE_SOURCE_DIR=`pwd`/moose-core if [ ! -d $MOOSE_SOURCE_DIR ]; then - git clone https://github.com/dilawar/moose-core -b $BRANCH --depth 10 + git clone https://github.com/BhallaLab/moose-core -b $BRANCH --depth 10 fi cd moose-core && git pull From a932a4bac3586fb801cf03c9881868ee9d8e47a5 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 03:42:28 +0530 Subject: [PATCH 10/31] temp commit. --- build_wheels_linux.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index f946d32..a05e1bc 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -41,6 +41,10 @@ if [ ! -f /usr/local/lib/libgsl.a ]; then cd gsl-2.4 CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS make install + + # GSL will be linked statically. + GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" + cd .. fi @@ -50,8 +54,6 @@ if [ ! -d $MOOSE_SOURCE_DIR ]; then git clone https://github.com/dilawar/moose-core --depth 10 --branch $BRANCH fi -# GSL will be linked statically. -GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" PY2=/opt/python/cp27-cp27m/bin/python2.7 $PY2 -m pip install numpy==1.14 matplotlib==2.2.4 From 46626ad8934c7f3b3bebca7a2768d0b0722ce9d3 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Sun, 11 Oct 2020 22:43:30 +0000 Subject: [PATCH 11/31] Fixes. Works in docker image now. --- build_wheels_linux.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 13f26e5..30cc569 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -51,7 +51,6 @@ fi # Try to link statically. GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" -CMAKE=/usr/bin/cmake3 # Build wheels here. PY27=$(ls /opt/python/cp27-cp27m/bin/python?.?) @@ -60,6 +59,10 @@ PY36=$(ls /opt/python/cp36-cp36m/bin/python?.?) PY37=$(ls /opt/python/cp37-cp37m/bin/python?.?) PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) +# install latest cmake using pip and its location to PATH +$PY38 -m pip install cmake --user +export PATH=/opt/python/cp38-cp38/bin:$PATH + for PYTHON in $PY38 $PY37 $PY36 $PY35 $PY27; do echo "========= Building using $PYTHON ..." $PYTHON -m pip install pip setuptools --upgrade From 83e5c76f06a916dc6f04037de6a123edca446237 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Sun, 11 Oct 2020 22:44:23 +0000 Subject: [PATCH 12/31] Disable py27 and py35 --- build_wheels_linux.sh | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 30cc569..fb96be3 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -53,8 +53,8 @@ fi GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" # Build wheels here. -PY27=$(ls /opt/python/cp27-cp27m/bin/python?.?) -PY35=$(ls /opt/python/cp35-cp35m/bin/python?.?) +#PY27=$(ls /opt/python/cp27-cp27m/bin/python?.?) +#PY35=$(ls /opt/python/cp35-cp35m/bin/python?.?) PY36=$(ls /opt/python/cp36-cp36m/bin/python?.?) PY37=$(ls /opt/python/cp37-cp37m/bin/python?.?) PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) @@ -63,19 +63,12 @@ PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) $PY38 -m pip install cmake --user export PATH=/opt/python/cp38-cp38/bin:$PATH -for PYTHON in $PY38 $PY37 $PY36 $PY35 $PY27; do +for PYTHON in $PY38 $PY37 $PY36; do echo "========= Building using $PYTHON ..." $PYTHON -m pip install pip setuptools --upgrade - if [[ "$PYV" -eq "27" ]]; then - $PYTHON -m pip install numpy==1.15 - $PYTHON -m pip install matplotlib==2.2.3 - else - $PYTHON -m pip install numpy twine - $PYTHON -m pip install matplotlib - fi - + $PYTHON -m pip install numpy twine + $PYTHON -m pip install matplotlib $PYTHON -m pip install twine - # Removing existing pymoose if any. $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" From 9c5c6ef73d0a2c6c6ed5edd7359e69334654afcc Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 04:31:48 +0530 Subject: [PATCH 13/31] Updated a bit more. --- Makefile | 2 +- build_wheels_linux.sh | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 84d5e43..2f40485 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ wheels : ./Dockerfile ./build_wheels_linux.sh --build-arg PYPI_PASSWORD=$(PYPI_PASSWORD) . run : ./Dockerfile - docker run -it $(DOCKERFILE) bash + docker run -it quay.io/pypa/manylinux2010_x86_64 upload: docker push $(DOCKERFILE) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 27984e1..1138776 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -58,8 +58,6 @@ fi GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" # Build wheels here. -#PY27=$(ls /opt/python/cp27-cp27m/bin/python?.?) -#PY35=$(ls /opt/python/cp35-cp35m/bin/python?.?) PY36=$(ls /opt/python/cp36-cp36m/bin/python?.?) PY37=$(ls /opt/python/cp37-cp37m/bin/python?.?) PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) @@ -93,7 +91,7 @@ for PYTHON in $PY38 $PY37 $PY36; do ) done -$PY3 -m pip install twine auditwheel +$PY38 -m pip install twine auditwheel # List all wheels. ls -lh $WHEELHOUSE/*.whl @@ -106,10 +104,9 @@ for whl in $WHEELHOUSE/pymoose*.whl; do done echo "Installing before testing ... " -$PY2 -m pip install $WHEELHOUSE/pymoose-$VERSION-py2-none-any.whl -$PY3 -m pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +$PY38 -m pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl -for PY in $PY3 $PY2; do +for PY in $PY38; do $PY -c 'import moose; print(moose.__version__)' done From d6525bac2dae27eb1c6edfe478541c4db8c77054 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 04:50:44 +0530 Subject: [PATCH 14/31] Removed `sudo` as it has no effect on travis anymore. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6af3db5..ca04cd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,3 @@ -sudo : required - matrix: include: - os : osx @@ -7,7 +5,6 @@ matrix: - os: osx osx_image: xcode10 - os: linux - sudo: required services: - docker From b0babd06bf1252aee4f349764189f59759c5b6b9 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 05:20:27 +0530 Subject: [PATCH 15/31] Creating wheel --- build_wheels_linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 1138776..8d0546f 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -114,6 +114,6 @@ done cd $MOOSE_SOURCE_DIR rm -rf dist $PY38 setup.py sdist -$TWINE upload dist/pymoose*.tar.gz \ - --user bhallalab --password $PYMOOSE_PYPI_PASSWORD \ +$PY38 -m twine upload dist/pymoose*.tar.gz \ + --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ --skip-existing || echo "Failed to upload source distribution." From 429798cf8929c7c49c7ee70b4b8a04b9b8c55d02 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 05:38:40 +0530 Subject: [PATCH 16/31] Update a little bit more. --- build_wheels_linux.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 8d0546f..4985582 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -105,15 +105,14 @@ done echo "Installing before testing ... " $PY38 -m pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl - -for PY in $PY38; do - $PY -c 'import moose; print(moose.__version__)' -done +$PY38 -c 'import moose; print(moose.__version__)' # Now upload the source distribution. +set -e cd $MOOSE_SOURCE_DIR -rm -rf dist -$PY38 setup.py sdist -$PY38 -m twine upload dist/pymoose*.tar.gz \ - --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ - --skip-existing || echo "Failed to upload source distribution." +rm -rf dist && \ + $PY38 setup.py sdist && \ + $PY38 -m twine upload dist/pymoose*.tar.gz \ + --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ + --skip-existing || echo "Failed to upload source distribution." +set +e From 2634f92c937c42c1763ce0b20a35620c2e2c93ee Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 05:40:12 +0530 Subject: [PATCH 17/31] One more test. --- build_wheels_linux.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 4985582..5cc4774 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -95,6 +95,9 @@ $PY38 -m pip install twine auditwheel # List all wheels. ls -lh $WHEELHOUSE/*.whl +$PY38 -m twine upload $WHEELHOUSE/*.whl \ + --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ + --skip-existing # now check the wheels. for whl in $WHEELHOUSE/pymoose*.whl; do From 26cf8ed657d775b8309ae7807772e3da8807cac7 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 00:36:46 +0000 Subject: [PATCH 18/31] Updated a little bit more. --- build_wheels_linux.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 5cc4774..cf933c6 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -64,7 +64,7 @@ PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) # install latest cmake using pip and its location to PATH $PY38 -m pip install cmake --user -export PATH=/opt/python/cp38-cp38/bin:$PATH +export PATH=/opt/python/cp38-cp38/bin:/root/.local/bin:$PATH for PYTHON in $PY38 $PY37 $PY36; do echo "========= Building using $PYTHON ..." @@ -118,4 +118,3 @@ rm -rf dist && \ $PY38 -m twine upload dist/pymoose*.tar.gz \ --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ --skip-existing || echo "Failed to upload source distribution." -set +e From c511d6ff4020a07472d604616f2603b2388fd3b2 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 06:24:55 +0530 Subject: [PATCH 19/31] Use __token__ --- build_wheels_linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index cf933c6..5d7e1f3 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -96,7 +96,7 @@ $PY38 -m pip install twine auditwheel # List all wheels. ls -lh $WHEELHOUSE/*.whl $PY38 -m twine upload $WHEELHOUSE/*.whl \ - --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ + --user __token__ --password $PYMOOSE_PYPI_PASSWORD \ --skip-existing # now check the wheels. @@ -116,5 +116,5 @@ cd $MOOSE_SOURCE_DIR rm -rf dist && \ $PY38 setup.py sdist && \ $PY38 -m twine upload dist/pymoose*.tar.gz \ - --user dilawar --password $PYMOOSE_PYPI_PASSWORD \ + --user __token__ --password $PYMOOSE_PYPI_PASSWORD \ --skip-existing || echo "Failed to upload source distribution." From a0872351d903769260a055244ba0eb21d9021481 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 06:38:07 +0530 Subject: [PATCH 20/31] Using travis password. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f40485..f2da4e3 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ DOCKERFILE:="bhallalab/manylinux-moose:latest" wheels : ./Dockerfile ./build_wheels_linux.sh mkdir -p $(HOME)/wheelhouse docker build -t $(DOCKERFILE) \ - --build-arg PYPI_PASSWORD=$(PYPI_PASSWORD) . + --build-arg PYMOOSE_PYPI_PASSWORD=$(PYMOOSE_PYPI_PASSWORD) . run : ./Dockerfile docker run -it quay.io/pypa/manylinux2010_x86_64 From 5246d31c2c17cb71359047c6a8eeafa7557abcaa Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 07:07:43 +0530 Subject: [PATCH 21/31] __token__ is set --- Makefile | 2 +- build_wheels_osx.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f2da4e3..527f272 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all : wheels -DOCKERFILE:="bhallalab/manylinux-moose:latest" +DOCKERFILE:="dilawars/pymoose-manylinux2010:latest" wheels : ./Dockerfile ./build_wheels_linux.sh mkdir -p $(HOME)/wheelhouse diff --git a/build_wheels_osx.sh b/build_wheels_osx.sh index cb36f92..0bc5272 100755 --- a/build_wheels_osx.sh +++ b/build_wheels_osx.sh @@ -58,6 +58,6 @@ PLATFORM=$($PYTHON -c "import distutils.util; print(distutils.util.get_platform( if [ ! -z "$PYMOOSE_PYPI_PASSWORD" ]; then echo "Did you test the wheels? I am uploading anyway ..." - $PYTHON -m twine upload -u bhallalab -p $PYMOOSE_PYPI_PASSWORD \ + $PYTHON -m twine upload -u __token__ -p $PYMOOSE_PYPI_PASSWORD \ $WHEELHOUSE/pymoose*.whl || echo "Failed to upload to PyPi" fi From 5295a970e1e4b0b3a21b6fb15e4415bc2cf82251 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 07:20:26 +0530 Subject: [PATCH 22/31] Fixed env variable name in dockerfile. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7b3699d..b51f4b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM quay.io/pypa/manylinux2010_x86_64 MAINTAINER Dilawar Singh -ARG PYPI_PASSWORD +ARG PYMOOSE_PYPI_PASSWORD RUN yum install -y cmake3 git tree && rm -rf /var/cache/yum/* From 580fcf2f30c471a0040219e58345662fb02f9aa5 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 02:33:32 +0000 Subject: [PATCH 23/31] Fix wheel before uploading them to pypi --- build_wheels_linux.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 5d7e1f3..79ad684 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -64,6 +64,8 @@ PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) # install latest cmake using pip and its location to PATH $PY38 -m pip install cmake --user +$PY38 -m pip install twine auditwheel + export PATH=/opt/python/cp38-cp38/bin:/root/.local/bin:$PATH for PYTHON in $PY38 $PY37 $PY36; do @@ -81,17 +83,14 @@ for PYTHON in $PY38 $PY37 $PY36; do $PYTHON setup.py bdist_wheel --skip-build ( echo "Install and test this wheel" - # NOTE: Not sure why I have to do this. But cant install wheel from build - # directory. + auditwheel repair $MOOSE_SOURCE_DIR/dist/*.whl -w $WHEELHOUSE/ + $PYTHON -m pip install $MOOSE_SOURCE_DIR/dist/*.whl || echo "Failed to install" cd /tmp - $PYTHON -m pip install $MOOSE_SOURCE_DIR/dist/*.whl $PYTHON $TESTFILE - mv $MOOSE_SOURCE_DIR/dist/*.whl $WHEELHOUSE rm -rf $MOOSE_SOURCE_DIR/dist/*.whl ) done -$PY38 -m pip install twine auditwheel # List all wheels. ls -lh $WHEELHOUSE/*.whl From 730111230772613eb5efdd454579d31520256d0b Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 08:19:12 +0530 Subject: [PATCH 24/31] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index df4a767..8ee49b2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev +3.2.1-dev From 913d8b8ecf4c8f4e883b4369e688022d94966367 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 08:56:54 +0530 Subject: [PATCH 25/31] version is read from moose-core. --- build_wheels_linux.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 79ad684..a5396b3 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -20,7 +20,6 @@ mkdir -p $WHEELHOUSE # tag on github and revision number. Make sure that they are there. [ -f ./BRANCH ] || echo "master" > ./BRANCH BRANCH=$(cat ./BRANCH) -VERSION="3.2dev$(date +%Y%m%d)" # Create a test script and upload. TESTFILE=/tmp/test.py @@ -32,9 +31,6 @@ moose.reinit() moose.start( 1 ) EOF - -echo "Building version $VERSION, from branch $BRANCH" - if [ ! -f /usr/local/lib/libgsl.a ]; then curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz tar xvf gsl-2.4.tar.gz @@ -98,17 +94,6 @@ $PY38 -m twine upload $WHEELHOUSE/*.whl \ --user __token__ --password $PYMOOSE_PYPI_PASSWORD \ --skip-existing -# now check the wheels. -for whl in $WHEELHOUSE/pymoose*.whl; do - auditwheel show "$whl" - # Fix the tag and remove the old wheel. - auditwheel repair "$whl" -w $WHEELHOUSE && rm -f "$whl" -done - -echo "Installing before testing ... " -$PY38 -m pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl -$PY38 -c 'import moose; print(moose.__version__)' - # Now upload the source distribution. set -e cd $MOOSE_SOURCE_DIR From 80b3d713a5ea06213d8bcd3552f911af1186488c Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 08:57:50 +0530 Subject: [PATCH 26/31] builds from master branch. --- build_wheels_osx.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_wheels_osx.sh b/build_wheels_osx.sh index 0bc5272..378a7ed 100755 --- a/build_wheels_osx.sh +++ b/build_wheels_osx.sh @@ -20,15 +20,13 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" MOOSE_SOURCE_DIR=`pwd`/moose-core if [ ! -d $MOOSE_SOURCE_DIR ]; then - git clone https://github.com/BhallaLab/moose-core -b $BRANCH --depth 10 + git clone https://github.com/dilawar/moose-core -b $BRANCH --depth 10 fi cd moose-core && git pull WHEELHOUSE=$HOME/wheelhouse rm -rf $WHEELHOUSE && mkdir -p $WHEELHOUSE -# Current version 0.7.4 seems to be broken with python3.7 . -# See https://travis-ci.org/BhallaLab/deploy/jobs/435219820 /usr/local/bin/python3 -m pip install delocate DELOCATE_WHEEL=/usr/local/bin/delocate-wheel From 5f2525233ad56b189bbfdcef9fe4d30d296b2448 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 09:03:00 +0530 Subject: [PATCH 27/31] Delete VERSION version is read from setup.py file. --- VERSION | 1 - 1 file changed, 1 deletion(-) delete mode 100644 VERSION diff --git a/VERSION b/VERSION deleted file mode 100644 index 8ee49b2..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -3.2.1-dev From f2066dfdf1b481420df003ef653af5282f960109 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 09:04:34 +0530 Subject: [PATCH 28/31] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6429520..5a1cb3a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -[![Build -Status](https://travis-ci.org/dilawar/pymoose-deploy.svg?branch=master)](https://travis-ci.org/dilawar/pymoose-deploy) +[![Build Status](https://travis-ci.org/dilawar/pymoose-deploy.svg?branch=master)](https://travis-ci.org/dilawar/pymoose-deploy) This repository contains all scripts required to build and release pymoose. From b0205bbc7f191f0c3741de67bad9f726678ee2ee Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 09:36:37 +0530 Subject: [PATCH 29/31] Using multiple xcode. --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index ca04cd1..fde0fcb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,16 @@ matrix: include: + - os: osx + osx_image: xcode12 - os : osx osx_image: xcode11 - os: osx osx_image: xcode10 + - os: osx + osx_image: xcode9 - os: linux + services: - docker From 09df3e4a2a8de1b312ef23cdf521a25021b03e2d Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 09:40:29 +0530 Subject: [PATCH 30/31] don't update. --- build_wheels_osx.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build_wheels_osx.sh b/build_wheels_osx.sh index 378a7ed..cae2861 100755 --- a/build_wheels_osx.sh +++ b/build_wheels_osx.sh @@ -6,8 +6,6 @@ BRANCH=$(cat ./BRANCH) # Just to be sure on homebrew. export PATH=/usr/local/bin:$PATH - -brew update || echo "Failed to update brew" brew install gsl || brew upgrade gsl brew install python@3 || echo "Failed to install python3" From 89bb1522cc41a5340cdbe1a6e51a49feeab704b8 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 12 Oct 2020 14:52:29 +0530 Subject: [PATCH 31/31] removed xcode9. --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fde0fcb..f399df5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,6 @@ matrix: osx_image: xcode11 - os: osx osx_image: xcode10 - - os: osx - osx_image: xcode9 - os: linux