add code env image build from gh workflow #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Publish Docker Image | |
| on: | |
| push: | |
| branches: | |
| - master | |
| env: | |
| DSS_VERSION: 13.5.2 | |
| KUBE_NAMESPACE: dataiku | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Check if Docker image exists | |
| id: image-check | |
| run: | | |
| IMAGE_NAME="${{ secrets.DOCKER_USERNAME }}/dataiku:${{ env.DSS_VERSION }}" | |
| echo "Checking for image: $IMAGE_NAME" | |
| # Use docker manifest inspect to check if the image exists | |
| if docker manifest inspect "$IMAGE_NAME" > /dev/null 2>&1; then | |
| echo "Image exists" | |
| echo "::set-output name=image_exists::true" | |
| else | |
| echo "Image does not exist" | |
| echo "::set-output name=image_exists::false" | |
| fi | |
| - name: Set up Docker Buildx | |
| if: steps.image-check.outputs.image_exists == 'false' | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Cache Docker layers | |
| if: steps.image-check.outputs.image_exists == 'false' | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx- | |
| - name: Build and push Docker image | |
| if: steps.image-check.outputs.image_exists == 'false' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: dss-docker | |
| push: true | |
| tags: | | |
| ${{ secrets.DOCKER_USERNAME }}/dataiku:${{ env.DSS_VERSION }} | |
| ${{ secrets.DOCKER_USERNAME }}/dataiku:latest | |
| build-args: | | |
| dssVersion=${{ env.DSS_VERSION }} | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| cache-to: type=local,dest=/tmp/.buildx-cache,mode=max | |
| - name: Post cache Docker layers | |
| if: steps.image-check.outputs.image_exists == 'false' | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ github.sha }} | |
| deploy: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| environment: demo | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Authenticate to Kubernetes cluster | |
| uses: azure/k8s-set-context@v3 | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.KUBECONFIG }} | |
| - name: Deploy to Kubernetes | |
| run: kubectl apply -f kubernetes/dataiku.yaml | |
| - name: Wait for Dataiku pod to be ready | |
| run: | | |
| kubectl wait --for=condition=Ready pod -l app=dataiku -n ${{ env.KUBE_NAMESPACE }} --timeout=300s | |
| POD_NAME=$(kubectl get pods -n ${{ env.KUBE_NAMESPACE }} -l app=dataiku -o jsonpath='{.items[0].metadata.name}') | |
| echo "POD_NAME=$POD_NAME" >> $GITHUB_ENV | |
| - name: Wait for Dataiku instance to be ready | |
| run: | | |
| until kubectl exec -n ${{ env.KUBE_NAMESPACE }} $POD_NAME -- sh -c "cd dss && ./bin/dssadmin run-diagnosis -d /tmp/diag.zip"; do | |
| echo "Instance not ready yet, retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| echo "Instance is ready!" | |
| - name: Login to Scaleway Registry inside Dataiku pod | |
| run: | | |
| kubectl -n ${{ env.KUBE_NAMESPACE }} exec -it ${{ env.POD_NAME }} -- docker login ${{ secrets.DOCKER_REGISTRY }} -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Build and push base image | |
| run: | | |
| kubectl exec -it -n ${{ env.KUBE_NAMESPACE }} ${{ env.POD_NAME }} -- sh -c "cd dss;./bin/dssadmin build-base-image --type container-exec --with-py311 --with-py39 && docker tag \$(docker images | egrep '^dku-exec-base.*${{env.DSS_VERSION}}' | awk '{print \$1 "'"'":"'"'" \$2}') ${{ secrets.DOCKER_REGISTRY }}/\$(docker images | egrep '^dku-exec-base.*${{env.DSS_VERSION}}' | awk '{print \$1 "'"'":"'"'" \$2}') && docker push ${{ secrets.DOCKER_REGISTRY }}/\$(docker images | egrep '^dku-exec-base.*${{env.DSS_VERSION}}' | awk '{print \$1 "'"'":"'"'" \$2}')" | |
| - name: Build and push cde image | |
| run: | | |
| kubectl exec -it -n ${{ env.KUBE_NAMESPACE }} ${{ env.POD_NAME }} -- sh -c "cd dss;./bin/dssadmin build-base-image --type cde --with-py311 --with-py39 && docker tag \$(docker images | egrep '^dku-cde-base.*${{env.DSS_VERSION}}' | awk '{print \$1 "'"'":"'"'" \$2}') ${{ secrets.DOCKER_REGISTRY }}/\$(docker images | egrep '^dku-cde-base.*${{env.DSS_VERSION}}' | awk '{print \$1 "'"'":"'"'" \$2}') && docker push ${{ secrets.DOCKER_REGISTRY }}/\$(docker images | egrep '^dku-cde-base.*${{env.DSS_VERSION}}' | awk '{print \$1 "'"'":"'"'" \$2}')" | |
| - name: Check if code-env images exist | |
| id: code-env-check | |
| run: | | |
| # Vérifier si les images de code-env existent déjà dans le registry | |
| IMAGE_PREFIX="${{ secrets.DOCKER_REGISTRY }}/dku-code-env" | |
| VERSION="${{ env.DSS_VERSION }}" | |
| # Vérifier les images Python 3.9 et 3.11 | |
| if docker manifest inspect "${IMAGE_PREFIX}-python39:${VERSION}" > /dev/null 2>&1 && \ | |
| docker manifest inspect "${IMAGE_PREFIX}-python311:${VERSION}" > /dev/null 2>&1; then | |
| echo "Code-env images exist" | |
| echo "::set-output name=code_env_exists::true" | |
| else | |
| echo "Code-env images do not exist" | |
| echo "::set-output name=code_env_exists::false" | |
| fi | |
| - name: Build and push code-env images | |
| if: steps.code-env-check.outputs.code_env_exists == 'false' | |
| run: | | |
| # Construire les images de code-env | |
| kubectl exec -it -n ${{ env.KUBE_NAMESPACE }} ${{ env.POD_NAME }} -- sh -c "cd dss && ./bin/dssadmin build-container-exec-code-env-images --all" | |
| # Récupérer la liste des images construites et les pousser vers le registry | |
| kubectl exec -it -n ${{ env.KUBE_NAMESPACE }} ${{ env.POD_NAME }} -- sh -c " | |
| for img in \$(docker images | grep '^dku-code-env' | awk '{print \$1\":\"\$2}'); do | |
| echo \"Tagging and pushing \$img\" | |
| docker tag \$img ${{ secrets.DOCKER_REGISTRY }}/\$img | |
| docker push ${{ secrets.DOCKER_REGISTRY }}/\$img | |
| done | |
| " |