diff --git a/.travis.yml b/.travis.yml index 6af3db5..f399df5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,13 @@ -sudo : required - matrix: include: + - os: osx + osx_image: xcode12 - os : osx osx_image: xcode11 - os: osx osx_image: xcode10 - os: linux - sudo: required + services: - docker diff --git a/BuildImages2014/Dockerfile b/BuildImages2014/Dockerfile new file mode 100644 index 0000000..8d74e79 --- /dev/null +++ b/BuildImages2014/Dockerfile @@ -0,0 +1,15 @@ +FROM quay.io/pypa/manylinux2014_x86_64 +MAINTAINER Dilawar Singh + +ARG PYPI_PASSWORD + +MAINTAINER Dilawar Singh +ENV PATH=/usr/local/bin:$PATH +RUN yum install -y cmake3 wget vim git +WORKDIR /root +RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz && \ + tar xvf gsl*.tar.gz && \ + cd gsl* && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS && make install +COPY build_wheels_linux.sh . +# RUN ./build_wheels_linux.sh +CMD [ "bash" ] diff --git a/BuildImages2014/Makefile b/BuildImages2014/Makefile new file mode 100644 index 0000000..acb5f9b --- /dev/null +++ b/BuildImages2014/Makefile @@ -0,0 +1,7 @@ +IMAGE := bhallalab/manylinux + +all : Dockerfile + docker build -t $(IMAGE) . + +run : + docker run -it $(IMAGE) bash diff --git a/BuildImages2014/build_wheels_linux.sh b/BuildImages2014/build_wheels_linux.sh new file mode 100755 index 0000000..8982937 --- /dev/null +++ b/BuildImages2014/build_wheels_linux.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -e -x + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) +NUM_WORKERS=$((NPROC+1)) + +if [ "$TRAVIS" == "true" ]; then + NUM_WORKERS=3 +fi +MAKEOPTS="-j$NUM_WORKERS" + +# Place to store wheels. +WHEELHOUSE=${1-$HOME/wheelhouse} +echo "Path to store wheels : $WHEELHOUSE" +mkdir -p $WHEELHOUSE + +# tag on github and revision number. Make sure that they are there. +[ -f ./BRANCH ] || echo "master" >> ./BRANCH +BRANCH=$(cat ./BRANCH) +VERSION="3.2.0.dev$(date +%Y%m%d)" + +echo "Building version $REVISION, from branch $BRANCH" + +GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" + +if [ ! -f /usr/local/lib/libgsl.a ]; then + #wget --no-check-certificate ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz + curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz + tar xvf gsl-2.4.tar.gz + cd gsl-2.4 + CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS + make install + cd .. +fi + +MOOSE_SOURCE_DIR=$SCRIPT_DIR/moose-core + +if [ ! -d $MOOSE_SOURCE_DIR ]; then + git clone https://github.com/dilawar/moose-core --depth 10 --branch $BRANCH +fi + +# Try to link statically. +CMAKE=/usr/bin/cmake3 + +# Build wheels here. +for PYV in 38 37m 36m; do + PYDIR=/opt/python/cp${PYV}-cp${PYV} + PYVER=$(basename $PYDIR) + mkdir -p $PYVER + ( + cd $PYVER + echo "Building using $PYDIR in $PYVER" + PYTHON=$(ls $PYDIR/bin/python?.?) + $PYTHON -m pip install numpy twine + $PYTHON -m pip install matplotlib + $PYTHON -m pip install twine + $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" + git pull || echo "Failed to pull $BRANCH" + $CMAKE -DPYTHON_EXECUTABLE=$PYTHON \ + -DGSL_STATIC_LIBRARIES=$GSL_STATIC_LIBS \ + -DVERSION_MOOSE=$VERSION \ + ${MOOSE_SOURCE_DIR} + make $MAKEOPTS + + # Now build bdist_wheel + cd python + cp setup.cmake.py setup.py + $PYTHON -m pip wheel . -w $WHEELHOUSE + echo "Content of WHEELHOUSE" + ls -lh $WHEELHOUSE/*.whl + ) +done + +# List all wheels. +ls -lh $WHEELHOUSE/*.whl + +# now check the wheels. +for whl in $WHEELHOUSE/pymoose*.whl; do + auditwheel show "$whl" +done + +echo "Installing before testing ... " + +/opt/python/cp36-cp36/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +/opt/python/cp38-cp38/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl +for PYV in 38 36; do + /opt/python/cp${PYV}-cp${PYV}/bin/pip install $WHEELHOUSE/pymoose-$VERSION-py3-none-any.whl + PYDIR=/opt/python/cp${PYV}-cp${PYV} + PYTHON=$(ls $PYDIR/bin/python?.?) + $PYTHON -c 'import moose; print(moose.__version__)' +done diff --git a/Dockerfile b/Dockerfile index 0ec3c94..b51f4b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,21 @@ -# This docker image is based on bhallalab/manylinux2010 which is -# ./hub.docker.com/wheel/Makefile -FROM bhallalab/manylinux2010:latest +FROM quay.io/pypa/manylinux2010_x86_64 MAINTAINER Dilawar Singh + ARG PYMOOSE_PYPI_PASSWORD -ENV PYMOOSE_PYPI_PASSWORD=$PYMOOSE_PYPI_PASSWORD + +RUN yum install -y cmake3 git tree && rm -rf /var/cache/yum/* + +ENV PATH /usr/local/bin:$PATH +RUN ln -s /usr/bin/cmake3 /usr/local/bin/cmake + WORKDIR /root -COPY ./BRANCH . + +RUN curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz \ + && tar xvf gsl-2.4.tar.gz \ + && cd gsl-2.4 \ + && CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS \ + && make install + COPY ./build_wheels_linux.sh . -RUN PYMOOSE_PYPI_PASSWORD=$PYMOOSE_PYPI_PASSWORD ./build_wheels_linux.sh +RUN ./build_wheels_linux.sh +CMD [ "bash", "-c", "./build_wheels_linux.sh"] diff --git a/Makefile b/Makefile index 3c032ba..527f272 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ -VERSION:=$(shell cat ./VERSION) - all : wheels -DOCKERFILE:="bhallalab/pymoose_wheels_$(USER):$(VERSION)" +DOCKERFILE:="dilawars/pymoose-manylinux2010:latest" wheels : ./Dockerfile ./build_wheels_linux.sh mkdir -p $(HOME)/wheelhouse - docker build \ - -t $(DOCKERFILE) \ + docker build -t $(DOCKERFILE) \ --build-arg PYMOOSE_PYPI_PASSWORD=$(PYMOOSE_PYPI_PASSWORD) . -run: - docker run -it $(DOCKERFILE) /bin/bash +run : ./Dockerfile + docker run -it quay.io/pypa/manylinux2010_x86_64 + +upload: + docker push $(DOCKERFILE) diff --git a/README.md b/README.md index 6fe83f0..5a1cb3a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/BhallaLab/deploy.svg?branch=master)](https://travis-ci.org/BhallaLab/deploy) +[![Build Status](https://travis-ci.org/dilawar/pymoose-deploy.svg?branch=master)](https://travis-ci.org/dilawar/pymoose-deploy) This repository contains all scripts required to build and release pymoose. diff --git a/VERSION b/VERSION deleted file mode 100644 index df4a767..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -3.2.0-dev diff --git a/build_wheels_linux.sh b/build_wheels_linux.sh index 13f26e5..a5396b3 100755 --- a/build_wheels_linux.sh +++ b/build_wheels_linux.sh @@ -4,7 +4,11 @@ set -e -x SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" NPROC=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) -NUM_WORKERS=$((NPROC)) +NUM_WORKERS=$((NPROC-1)) + +if [ "$TRAVIS" == "true" ]; then + NUM_WORKERS=2 +fi MAKEOPTS="-j$NUM_WORKERS" # Place to store wheels. @@ -14,8 +18,8 @@ echo "Path to store wheels : $WHEELHOUSE" mkdir -p $WHEELHOUSE # tag on github and revision number. Make sure that they are there. +[ -f ./BRANCH ] || echo "master" > ./BRANCH BRANCH=$(cat ./BRANCH) -VERSION="3.2dev$(date +%Y%m%d)" # Create a test script and upload. TESTFILE=/tmp/test.py @@ -27,52 +31,45 @@ moose.reinit() moose.start( 1 ) EOF - -echo "Building version $VERSION, from branch $BRANCH" - if [ ! -f /usr/local/lib/libgsl.a ]; then - #wget --no-check-certificate ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz curl -O https://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz tar xvf gsl-2.4.tar.gz cd gsl-2.4 CFLAGS=-fPIC ./configure --enable-static && make $MAKEOPTS make install + + # GSL will be linked statically. + GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" + cd .. fi MOOSE_SOURCE_DIR=$SCRIPT_DIR/moose-core -if [ -d $MOOSE_SOURCE_DIR ]; then - cd $MOOSE_SOURCE_DIR && git checkout $BRANCH && git pull origin $BRANCH - rm -rf dist -else - git clone https://github.com/dilawar/moose-core $MOOSE_SOURCE_DIR \ - --depth 1 --branch $BRANCH + +if [ ! -d $MOOSE_SOURCE_DIR ]; then + git clone https://github.com/dilawar/moose-core --depth 10 --branch $BRANCH fi # Try to link statically. GSL_STATIC_LIBS="/usr/local/lib/libgsl.a;/usr/local/lib/libgslcblas.a" -CMAKE=/usr/bin/cmake3 # Build wheels here. -PY27=$(ls /opt/python/cp27-cp27m/bin/python?.?) -PY35=$(ls /opt/python/cp35-cp35m/bin/python?.?) PY36=$(ls /opt/python/cp36-cp36m/bin/python?.?) PY37=$(ls /opt/python/cp37-cp37m/bin/python?.?) PY38=$(ls /opt/python/cp38-cp38/bin/python?.?) -for PYTHON in $PY38 $PY37 $PY36 $PY35 $PY27; do +# install latest cmake using pip and its location to PATH +$PY38 -m pip install cmake --user +$PY38 -m pip install twine auditwheel + +export PATH=/opt/python/cp38-cp38/bin:/root/.local/bin:$PATH + +for PYTHON in $PY38 $PY37 $PY36; do echo "========= Building using $PYTHON ..." $PYTHON -m pip install pip setuptools --upgrade - if [[ "$PYV" -eq "27" ]]; then - $PYTHON -m pip install numpy==1.15 - $PYTHON -m pip install matplotlib==2.2.3 - else - $PYTHON -m pip install numpy twine - $PYTHON -m pip install matplotlib - fi - + $PYTHON -m pip install numpy twine + $PYTHON -m pip install matplotlib $PYTHON -m pip install twine - # Removing existing pymoose if any. $PYTHON -m pip uninstall pymoose -y || echo "No pymoose" @@ -82,44 +79,26 @@ for PYTHON in $PY38 $PY37 $PY36 $PY35 $PY27; do $PYTHON setup.py bdist_wheel --skip-build ( echo "Install and test this wheel" - # NOTE: Not sure why I have to do this. But cant install wheel from build - # directory. + auditwheel repair $MOOSE_SOURCE_DIR/dist/*.whl -w $WHEELHOUSE/ + $PYTHON -m pip install $MOOSE_SOURCE_DIR/dist/*.whl || echo "Failed to install" cd /tmp - $PYTHON -m pip install $MOOSE_SOURCE_DIR/dist/*.whl $PYTHON $TESTFILE - mv $MOOSE_SOURCE_DIR/dist/*.whl $WHEELHOUSE rm -rf $MOOSE_SOURCE_DIR/dist/*.whl ) done + # List all wheels. ls -lh $WHEELHOUSE/*.whl - -# now check the wheels. -for whl in $WHEELHOUSE/pymoose*.whl; do - auditwheel show "$whl" - # Fix the tag and remove the old wheel. - auditwheel repair "$whl" -w $WHEELHOUSE && rm -f "$whl" -done - -# upload to PYPI. -$PY38 -m pip install twine -TWINE="$PY38 -m twine" -for whl in `find $WHEELHOUSE -name "pymoose*.whl"`; do - # If successful, upload using twine. - if [ -n "$PYMOOSE_PYPI_PASSWORD" ]; then - $TWINE upload $whl \ - --user bhallalab \ - --password $PYMOOSE_PYPI_PASSWORD --skip-existing - else - echo "PYPI password is not set" - fi -done +$PY38 -m twine upload $WHEELHOUSE/*.whl \ + --user __token__ --password $PYMOOSE_PYPI_PASSWORD \ + --skip-existing # Now upload the source distribution. +set -e cd $MOOSE_SOURCE_DIR -rm -rf dist -$PY38 setup.py sdist -$TWINE upload dist/pymoose*.tar.gz \ - --user bhallalab --password $PYMOOSE_PYPI_PASSWORD \ - --skip-existing || echo "Failed to upload source distribution." +rm -rf dist && \ + $PY38 setup.py sdist && \ + $PY38 -m twine upload dist/pymoose*.tar.gz \ + --user __token__ --password $PYMOOSE_PYPI_PASSWORD \ + --skip-existing || echo "Failed to upload source distribution." diff --git a/build_wheels_osx.sh b/build_wheels_osx.sh index e5c46a3..cae2861 100755 --- a/build_wheels_osx.sh +++ b/build_wheels_osx.sh @@ -6,11 +6,8 @@ BRANCH=$(cat ./BRANCH) # Just to be sure on homebrew. export PATH=/usr/local/bin:$PATH - -brew update || echo "Failed to update brew" brew install gsl || brew upgrade gsl brew install python@3 || echo "Failed to install python3" -brew install python@2 || echo "Failed to install python2" # Following are to remove numpy; It is breaking the build on Xcode9.4. # brew uninstall gdal postgis || echo "Failed to uninstall gdal/postgis" @@ -28,39 +25,35 @@ cd moose-core && git pull WHEELHOUSE=$HOME/wheelhouse rm -rf $WHEELHOUSE && mkdir -p $WHEELHOUSE -# Current version 0.7.4 seems to be broken with python3.7 . -# See https://travis-ci.org/BhallaLab/deploy/jobs/435219820 /usr/local/bin/python3 -m pip install delocate DELOCATE_WHEEL=/usr/local/bin/delocate-wheel # Always prefer brew version. -for _py in 3 2; do - PYTHON=/usr/local/bin/python$_py - - if [ ! -f $PYTHON ]; then - echo "Not found $PYTHON" - continue - fi - +PYTHON=/usr/local/bin/python3 - $PYTHON -m pip install setuptools --upgrade --user - $PYTHON -m pip install wheel --upgrade --user - $PYTHON -m pip install numpy --upgrade --user - $PYTHON -m pip install twine --upgrade --user +if [ ! -f $PYTHON ]; then + echo "Not found $PYTHON" + continue +fi - PLATFORM=$($PYTHON -c "import distutils.util; print(distutils.util.get_platform())") - ( - cd $MOOSE_SOURCE_DIR - $PYTHON setup.py build_ext - export GSL_USE_STATIC_LIBRARIES=1 - $PYTHON setup.py bdist_wheel --skip-build - $DELOCATE_WHEEL -v dist/*.whl -w $WHEELHOUSE - rm -rf dist/*.whl - ) - if [ ! -z "$PYMOOSE_PYPI_PASSWORD" ]; then - echo "Did you test the wheels? I am uploading anyway ..." - $PYTHON -m twine upload -u bhallalab -p $PYMOOSE_PYPI_PASSWORD \ - $WHEELHOUSE/pymoose*.whl || echo "Failed to upload to PyPi" - fi -done +$PYTHON -m pip install setuptools --upgrade --user +$PYTHON -m pip install wheel --upgrade --user +$PYTHON -m pip install numpy --upgrade --user +$PYTHON -m pip install twine --upgrade --user + +PLATFORM=$($PYTHON -c "import distutils.util; print(distutils.util.get_platform())") +( + cd $MOOSE_SOURCE_DIR + $PYTHON setup.py build_ext + export GSL_USE_STATIC_LIBRARIES=1 + $PYTHON setup.py bdist_wheel --skip-build + $DELOCATE_WHEEL -v dist/*.whl -w $WHEELHOUSE + rm -rf dist/*.whl +) + +if [ ! -z "$PYMOOSE_PYPI_PASSWORD" ]; then + echo "Did you test the wheels? I am uploading anyway ..." + $PYTHON -m twine upload -u __token__ -p $PYMOOSE_PYPI_PASSWORD \ + $WHEELHOUSE/pymoose*.whl || echo "Failed to upload to PyPi" +fi diff --git a/hub.docker.com/Dockerfile b/hub.docker.com/Dockerfile new file mode 100644 index 0000000..f07ce35 --- /dev/null +++ b/hub.docker.com/Dockerfile @@ -0,0 +1,10 @@ +# Use the python3.7 image. +FROM python:3.7.6-slim-buster +MAINTAINER Dilawar Singh + +RUN apt update +RUN apt install -y python3-scipy python3-networkx graphviz \ + && rm -rf /var/lib/apt/lists/* +RUN python3 -m pip --no-cache-dir install pymoose --pre +RUN python3 -m pip --no-cache-dir install matplotlib numpy +CMD [ "bash" ] diff --git a/hub.docker.com/Dockerfile.moose b/hub.docker.com/Dockerfile.moose new file mode 100644 index 0000000..da4b9c9 --- /dev/null +++ b/hub.docker.com/Dockerfile.moose @@ -0,0 +1,29 @@ +FROM ubuntu:16.04 +MAINTAINER Dilawar Singh + +RUN apt update +RUN apt install -y wget gnupg2 sudo +RUN wget -nv https://download.opensuse.org/repositories/home:moose/xUbuntu_16.04/Release.key -O Release.key +RUN apt-key add - < Release.key +RUN sh -c "echo 'deb http://download.opensuse.org/repositories/home:/moose/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/home:moose.list" +RUN apt-get update +RUN apt install -y --allow-unauthenticated \ + python-matplotlib python-scipy python-qt4 python-networkx \ + moose moogli python-opengl python-qt4-gl graphviz xterm \ + && rm -rf /var/lib/apt/lists/* + +# Replace 1000 with your user / group id +RUN export uid=1000 gid=1000 && \ + mkdir -p /home/mooser && \ + echo "mooser:x:${uid}:${gid}:Mooser,,,:/home/mooser:/bin/bash" >> /etc/passwd && \ + echo "mooser:x:${uid}:" >> /etc/group && \ + echo "mooser ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/mooser && \ + chmod 0440 /etc/sudoers.d/mooser && \ + chown ${uid}:${gid} -R /home/mooser + +USER mooser +ENV HOME /home/mooser +# see https://github.com/osrf/docker_images/issues/21 +ENV QT_X11_NO_MITSHM=1 +CMD cd +CMD /usr/bin/xterm diff --git a/hub.docker.com/Makefile b/hub.docker.com/Makefile new file mode 100644 index 0000000..6f3a1af --- /dev/null +++ b/hub.docker.com/Makefile @@ -0,0 +1,19 @@ +all : moose pymoose + +moose : Dockerfile.moose + docker build -t bhallalab/moose:3.1.4 -f Dockerfile.moose . + docker build -t bhallalab/moose:latest -f Dockerfile.moose . + +pymoose : Dockerfile + docker build -t bhallalab/pymoose:3.2.git -f Dockerfile . + docker build -t bhallalab/pymoose:latest -f Dockerfile . + + +run_pymoose: + docker run -it bhallalab/pymoose:latest + +upload: + docker push bhallalab/pymoose:3.2.git + docker push bhallalab/pymoose:latest + docker push bhallalab/moose:3.1.4 + docker push bhallalab/moose:latest diff --git a/hub.docker.com/build_docker.sh b/hub.docker.com/build_docker.sh new file mode 100755 index 0000000..a7c4f56 --- /dev/null +++ b/hub.docker.com/build_docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t bhallalab/moose:3.1.4 . diff --git a/hub.docker.com/launch_docker.sh b/hub.docker.com/launch_docker.sh new file mode 100755 index 0000000..3cdbdb6 --- /dev/null +++ b/hub.docker.com/launch_docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +xhost + +docker run -ti --rm \ + -e DISPLAY=${DISPLAY} \ + -v /tmp/.X11-unix:/tmp/.X11-unix \ + bhallalab/moose:latest