diff --git a/.tool-versions b/.tool-versions index c6a0d037..eaebb597 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ java adoptopenjdk-8.0.265+1 +sbt 1.9.7 diff --git a/Jenkinsfile b/Jenkinsfile index aa517dec..d5b4c037 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,284 +1,34 @@ -@Library('socrata-pipeline-library@9.7.0') +@Library('socrata-pipeline-library@sarahs/sbt-library-publishing') _ -import com.socrata.ReleaseMetadataService -def rmsSupportedEnvironment = com.socrata.ReleaseMetadataService.SupportedEnvironment - -// set up service and project variables -String service = 'data-coordinator' -String project_wd = 'coordinator' -boolean isPr = env.CHANGE_ID != null -boolean isHotfix = isHotfixBranch(env.BRANCH_NAME) -boolean skip = false -String lastStage -boolean publishLibrary = false - -// instanciate libraries -def sbtbuild = new com.socrata.SBTBuild(steps, service, project_wd) -def dockerize = new com.socrata.Dockerize(steps, service, BUILD_NUMBER) -def releaseTag = new com.socrata.ReleaseTag(steps, service) -def semVerTag = new com.socrata.SemVerTag(steps) - -pipeline { - options { - ansiColor('xterm') - buildDiscarder(logRotator(numToKeepStr: '50')) - timeout(time: 20, unit: 'MINUTES') - } - parameters { - string(name: 'AGENT', defaultValue: 'build-worker-pg13', description: 'Which build agent to use?') - string(name: 'BRANCH_SPECIFIER', defaultValue: 'origin/main', description: 'Use this branch for building the artifact.') - booleanParam(name: 'RELEASE_BUILD', defaultValue: false, description: 'Are we building a release candidate?') - booleanParam(name: 'RELEASE_DRY_RUN', defaultValue: false, description: 'To test out the release build.') - string(name: 'RELEASE_NAME', description: 'For release builds, the release name which is used in the git tag and the build id.') - } - agent { - label params.AGENT - } - environment { - SCALA_VERSION = '2.12' - DEPLOY_PATTERN = "${service}*" - WEBHOOK_ID = 'WORKFLOW_EGRESS_AUTOMATION' - } - stages { - stage('Check for Version Change') { - when { - branch 'main' - not { expression { params.RELEASE_BUILD } } - } - steps { - script { - lastStage = env.STAGE_NAME - // Checkout to fetch tags because the default checkout does not include tags - // Fetching the tags without uisng checkout will confuse Jenkins about the git branch - semVerTag.checkout('main') - publishLibrary = haveCertainFilesChanged(filePaths: ['version.sbt']) - } - } - } - stage('Publish Library') { - when { - branch 'main' - not { expression { params.RELEASE_BUILD } } - expression { publishLibrary } - } - steps { - script { - lastStage = env.STAGE_NAME - skip = true - semVerTag.checkoutClosestTag() - // Build & Publish - sbtbuild.setRunITTest(true) - sbtbuild.setNoSubproject(true) - sbtbuild.setScalaVersion(env.SCALA_VERSION) - sbtbuild.setPublish(true) - sbtbuild.setBuildType("library") - sbtbuild.build() - } - } - } - stage('Hotfix Tag') { - when { - expression { isHotfix } - } - steps { - script { - lastStage = env.STAGE_NAME - if (releaseTag.noCommitsOnHotfixBranch(env.BRANCH_NAME)) { - skip = true - echo "SKIP: Skipping the rest of the build because there are no commits on the hotfix branch yet" - } else { - env.CURRENT_RELEASE_NAME = releaseTag.getReleaseName(env.BRANCH_NAME) - env.HOTFIX_NAME = releaseTag.getHotfixName(env.CURRENT_RELEASE_NAME) - } - } - } - } - stage('Generate Leaked Secrets Report') { - when { - not { expression { skip } } - } - steps { - script { - lastStage = env.STAGE_NAME - assert isInstalled('gitleaks'): 'gitleaks is missing.' - String secretsReportFileName = 'gitleaks-report.json' - String gitleaksCommand = getGitleaksCommand secretsReportFileName - assert sh (script: gitleaksCommand, returnStatus: true) == 0: \ - 'Attempt to run gitleaks failed.' - echo "Generated report ${secretsReportFileName}." - archiveArtifacts artifacts: secretsReportFileName, fingerprint: true - } - } - } - stage('Build') { - when { - allOf { - not { expression { return params.PUBLISH } } - not { expression { skip } } - } - } - steps { - script { - lastStage = env.STAGE_NAME - sbtbuild.setRunITTest(true) - sbtbuild.setNoSubproject(true) - sbtbuild.setPublish(false) - sbtbuild.setScalaVersion(env.SCALA_VERSION) - sbtbuild.setSrcJar("coordinator/target/coordinator-assembly.jar") - sbtbuild.build() - } - } - } - stage('Docker Build') { - when { - allOf { - not { expression { isPr } } - not { expression { skip } } - not { expression { return params.PUBLISH } } - } - } - steps { - script { - lastStage = env.STAGE_NAME - if (params.RELEASE_BUILD || isHotfix) { - env.VERSION = (isHotfix) ? env.HOTFIX_NAME : params.RELEASE_NAME - env.DOCKER_TAG = dockerize.dockerBuildWithSpecificTag( - tag: env.VERSION, - path: sbtbuild.getDockerPath(), - artifacts: [sbtbuild.getDockerArtifact()] - ) - } else { - env.DOCKER_TAG = dockerize.dockerBuildWithDefaultTag( - version: 'STAGING', - sha: env.GIT_COMMIT, - path: sbtbuild.getDockerPath(), - artifacts: [sbtbuild.getDockerArtifact()] - ) - } - } - } - post { - success { - script { - if (isHotfix || params.RELEASE_BUILD) { - String version = (isHotfix) ? env.HOTFIX_NAME : params.RELEASE_NAME - String branchName = (isHotfix) ? env.BRANCH_NAME : params.BRANCH_SPECIFIER - env.GIT_TAG = releaseTag.createWithCheckout(version, branchName, params.RELEASE_DRY_RUN) - } - if (params.RELEASE_BUILD) { - def releaseHelper = new com.socrata.ReleaseHelper(steps) - String previousPreviousRelease = releaseHelper.previous( - releaseHelper.previous( - releaseHelper.getMostRecentReleaseID() - ) - ) - releaseTag.createHotfixBranch(params.RELEASE_NAME, params.RELEASE_DRY_RUN) - releaseTag.cleanUpHotfixBranch(previousPreviousRelease, params.RELEASE_DRY_RUN) - } - } - } - } - } - stage('Publish Image') { - when { - allOf { - not { expression { isPr } } - not { expression { skip } } - not { expression { return params.PUBLISH } } - not { expression { return params.RELEASE_BUILD && params.RELEASE_DRY_RUN } } - } - } - steps { - script { - lastStage = env.STAGE_NAME - if (isHotfix || params.RELEASE_BUILD) { - env.BUILD_ID = dockerize.publish(sourceTag: env.DOCKER_TAG) - } else { - env.BUILD_ID = dockerize.publish( - sourceTag: env.DOCKER_TAG, - environments: ['internal'] - ) - } - currentBuild.description = env.BUILD_ID - } - } - post { - success { - script { - if (isHotfix || params.RELEASE_BUILD) { - env.PURPOSE = (isHotfix) ? 'hotfix' : 'initial' - env.RELEASE_ID = (isHotfix) ? env.CURRENT_RELEASE_NAME : params.RELEASE_NAME - Map buildInfo = [ - "project_id": service, - "build_id": env.BUILD_ID, - "release_id": env.RELEASE_ID, - "git_tag": env.GIT_TAG, - "purpose": env.PURPOSE, - ] - createBuild( - buildInfo, - rmsSupportedEnvironment.production - ) - } - } - } - } - } - stage('Deploy') { - when { - not { expression { isPr } } - not { expression { skip } } - not { expression { return params.RELEASE_BUILD } } - not { expression { return params.PUBLISH } } - } - steps { - script { - lastStage = env.STAGE_NAME - env.ENVIRONMENT = (isHotfix) ? 'rc' : 'staging' - marathonDeploy( - serviceName: env.DEPLOY_PATTERN, - tag: env.BUILD_ID, - environment: env.ENVIRONMENT - ) - // While working on migrating from marathon to ECS, we are keeping the tagged images up to date - // Once the migration is done, we will remove the marathonDeploy and leave in place this publish which triggers the ECS deployment - env.TARGET_DEPLOY_TAG = (env.ENVIRONMENT == 'rc') ? 'rc' : 'latest' - dockerize.publish( - sourceTag: env.DOCKER_TAG, - targetTag: env.TARGET_DEPLOY_TAG, - environments: ['internal'] - ) - } - } - post { - success { - script { - if (isHotfix) { - Map deployInfo = [ - "build_id": env.BUILD_ID, - "environment": env.ENVIRONMENT, - ] - createDeployment( - deployInfo, - rmsSupportedEnvironment.production - ) - } - } - } - } - } - } - post { - failure { - script { - boolean buildingMain = (env.JOB_NAME.contains("${service}/main")) - if (buildingMain) { - teamsWorkflowMessage( - message: "[${currentBuild.fullDisplayName}](${env.BUILD_URL}) has failed in stage ${lastStage}", - workflowCredentialID: WEBHOOK_ID - ) - } - } - } - } -} +commonPipeline( + defaultBuildWorker: 'build-worker-pg13', + jobName: 'data-coordinator', + language: 'scala', + languageOptions: [ + isMultiProjectRepository: true, + ], + numberOfBuildsToKeep: 50, + projects: [ + [ + name: 'data-coordinator', + compiled: true, + deploymentEcosystem: 'marathon-mesos', + docker: [ + buildContext: 'coordinator/docker', + ], + marathonInstanceNamePattern: 'data-coordinator*', + type: 'service', + ], + [ + name: 'coordinator-external', + compiled: true, + paths: [ + buildContext: '.', + version: 'version.sbt', + ], + type: 'library', + ], + ], + teamsChannelWebhookId: 'WORKFLOW_EGRESS_AUTOMATION', +) +// TODO: support sbtbuild.setRunITTest(true) = sbt it:test diff --git a/build.sbt b/build.sbt index 32b566af..32dd31ca 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ ThisBuild / scalacOptions ++= Seq("-deprecation", "-feature") ThisBuild / evictionErrorLevel := Level.Warn -val coordinatorExternal = (project in file("coordinator-external")). +val `coordinator-external` = (project in file("coordinator-external")). configs(IntegrationTest). settings(Defaults.itSettings) @@ -18,8 +18,8 @@ val coordinatorlib = (project in file("coordinatorlib")). configs(IntegrationTest). settings(Defaults.itSettings) -val coordinator = (project in file("coordinator")). - dependsOn(coordinatorlib, coordinatorExternal) +val `data-coordinator` = (project in file("coordinator")). + dependsOn(coordinatorlib, `coordinator-external`) val secondarylib = (project in file("secondarylib")). dependsOn(coordinatorlib) @@ -31,7 +31,7 @@ val secondarylibFeedback = (project in file("secondarylib-feedback")). dependsOn(secondarylib) val datasetMover = (project in file("dataset-mover")). - dependsOn(coordinator) + dependsOn(`data-coordinator`) publish / skip := true diff --git a/coordinator/build.sbt b/coordinator/build.sbt index 605b591e..abe1a693 100644 --- a/coordinator/build.sbt +++ b/coordinator/build.sbt @@ -1,6 +1,6 @@ import Dependencies._ -name := "coordinator" +name := "data-coordinator" libraryDependencies ++= Seq( c3po, diff --git a/coordinator/docker/Dockerfile b/coordinator/docker/Dockerfile index c0fa79f3..d5602b32 100644 --- a/coordinator/docker/Dockerfile +++ b/coordinator/docker/Dockerfile @@ -8,7 +8,7 @@ ENV JMX_PORT 6029 RUN apt-get -y update && apt-get -y install jq ENV SERVER_ROOT /srv/data-coordinator -ENV SERVER_ARTIFACT coordinator-assembly.jar +ENV SERVER_ARTIFACT data-coordinator-assembly.jar ENV SERVER_CONFIG data-coordinator.conf # defaults diff --git a/coordinator/docker/README.md b/coordinator/docker/README.md index 1b2ce20c..59e10d8a 100644 --- a/coordinator/docker/README.md +++ b/coordinator/docker/README.md @@ -1,7 +1,6 @@ # Docker support -The files in this directory allow you to build a docker image. The data coordinator assembly must be -copied to `coordinator-assembly.jar` in this directory before building. +The files in this directory allow you to build a docker image. The data coordinator assembly must be copied to `data-coordinator-assembly.jar` in this directory before building. ## Required Runtime Variables diff --git a/version.sbt b/version.sbt index ae6a86fc..1a91341e 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "4.2.46-SNAPSHOT" +ThisBuild / version := "4.2.50-SNAPSHOT"