build.gradle

/*
 * Gradle build for Jython. See also settings.gradle.
 *
 * This is an attempt to build a distributable JAR using Gradle that could be
 * cited as a dependency by other Gradle or Maven projects, when they use the
 * Jython interpreter from Java (under JSR-223 or directly).
 *
 * At present, the build is additional to the Ant build that remains the
 * primary and reliable support for development, for test, and to build the
 * Jython installers.
 *
 * The delivered jar should contain only Jython project material (Java classes
 * and the Python library) while the many JARs Jython depends on will be cited
 * in the accompanying metadata as dependencies.
 *
 * The Jython source structure does not follow the standard convention for
 * Gradle. This script deals with that without changing it, but it uses a build
 * directory (build2) entirely separate from Ant's, in which generated and
 * compiled material is posted conformant with Gradle conventions. This means
 * that the later tasks Gradle provides (test and jar) do not need so much
 * customisation.
 */

plugins {
  id 'java-library'
  id 'antlr'
  id 'maven-publish'
}


repositories {
  mavenCentral()
}


buildDir = file('build2')
ext {
  buildDate = new Date()
  antlrSrcDir = "$buildDir/gensrc/"
  antlrGenDir = "$antlrSrcDir/org/python/antlr"
  // Intermediate locations for compiled classes
  unexposedDir = "$buildDir/unexposed"
  exposedDir = "$buildDir/exposed"
  // The standard library may safely be assembled in-place as a resource
  pythonLibDir = "$buildDir/python/Lib/"
  buildLibDir = "$buildDir/resources/main/Lib/"
  buildTestLibDir = "$buildDir/resources/test/Lib/"
  compiledLibDir = "$buildDir/resources/main/Lib/"
  compiledTestLibDir = "$buildDir/resources/test/Lib/"
  // test folders
  libPython = 'lib-python/2.7'
  libJython = 'Lib'
  libTestSpecs = [
    'distutils/tests/',
    'email/test/',
    'json/tests/',
    'lib2to3/tests/',
    'unittest/test/',
    'test/'
  ]
}


java {
  sourceCompatibility = JavaVersion.VERSION_21
  withSourcesJar()
  withJavadocJar()
}


import java.text.SimpleDateFormat


// update the version to your needs 
// generateVersionInfo() will eventually add a '-SNAPSHOT' (but too late for gradle publication)
version = '2.7.5a1'
group = 'org.python'


dependencies {
  antlr libs.antlr

  compileOnly(libs.informix)
  compileOnly(libs.oracle)

  implementation(libs.ant)
  implementation(libs.asm)
  implementation(libs.asm.commons)
  implementation(libs.asm.util)
  implementation(libs.bcpkix.jdk18on)
  implementation(libs.bcprov.jdk18on)
  implementation(libs.commons.compress)
  implementation(libs.commons.io)
  implementation(libs.guava)
  implementation(libs.icu4j)
  implementation(libs.jakarta.servlet.api)
  implementation(libs.java.sizeof)
  implementation(libs.jffi)
  implementation(libs.jline)
  implementation(libs.jnr.constants)
  implementation(libs.jnr.ffi)
  implementation(libs.jnr.netdb)
  implementation(libs.jnr.posix)
  implementation(libs.netty.buffer)
  implementation(libs.netty.codec.base)
  implementation(libs.netty.common)
  implementation(libs.netty.handler)
  implementation(libs.netty.resolver)
  implementation(libs.netty.transport)

  testImplementation(libs.junit)
}


sourceSets {
  main{
    antlr {
      srcDirs = ['grammar']
      exclude 'Base.g' // Not used (and produces errors)
    }
    java {
      srcDirs = ['src', antlrSrcDir]
    }
  }

  test {  // Non-standard locations must be specified explicitly
    java {
      srcDirs = ['tests/java']
      // Reference to proprietary libraries not supplied
      exclude 'com/ziclix/python/sql/**'
    }
  }
}


generateGrammarSource {
  outputDirectory = file(antlrGenDir)
}


sourcesJar {
  dependsOn generateGrammarSource
}


compileJava {
  dependsOn generateGrammarSource
  // Divert compiled classes to intermediate location pre-exposure.
  destinationDirectory = file(unexposedDir)

  // should only warn about finalization and PythonParser
  options.compilerArgs = ['-Xlint:deprecation']

  // when activating unchecked, please make sure that only (more )PythonParser and finalization warnings pop up
//  options.compilerArgs = ['-Xlint:deprecation', '-Xlint:unchecked']
}


// ---------------- Resource Processing ----------------------------------------

/*
 * Jython brings several files we could treat as resources, but they do not sit
 * in the Gradle-conventional 'main/resources' directory, rather are in the
 * project root or rub shoulders with the java source. Pick them individually.
 *
 * Several tasks defined below declare that processResources depends on them,
 * with the objective that at the end of processResources all generated
 * resources and the stdlib (but not the compiled stdlib) should be in place
 * in $buildDir/resources/main.
 */
processResources {
  from(file('.')) {
    include 'LICENSE.txt'
  }
  from(file('src')) {
    include 'META-INF/**'
    include 'org/python/modules/ucnhash.dat'
    include 'com/ziclix/python/sql/resource/zxJDBCMessages.properties'
  }
}

// ---------------- Expose Task ------------------------------------------------

/*
 * The exposer operates between the output of compileJava (unexposed directory)
 * and a second intermediate location (exposed directory). These two the
 * mergeExposed task will finally combine in the Gradle-standard classes
 * directory used as input by the jar task.
 */
configurations {
  expose.extendsFrom(implementation)
}

dependencies {
  // The expose (Ant) task depends on classes compiled to here:
  expose files(unexposedDir)
}

// A (Gradle) task to run the Ant task 'expose'.
task expose (group: 'Custom', dependsOn: compileJava) {

  description = 'Expose Java types to Python using their annotations.'

  // Allow Gradle to infer the need to regenreate the outputs
  inputs.files(fileTree("${project.ext.unexposedDir}/org/python"))
  outputs.dir(project.ext.exposedDir)

  def exposedDir = project.ext.exposedDir
  def unexposedDir = project.ext.unexposedDir

  doLast {
    /*
     * Define an Ant task called 'expose' in the project's AntBuilder.
     * We can't define it until ExposeTask has been compiled.
     */
    ant.taskdef(
      name: 'expose',
      classname: 'org.python.expose.generate.ExposeTask',
      classpath: configurations.expose.asPath
     )

    // Use the Gradle-conventional directory structure (not the legacy one).
    ant.expose(
      srcdir: file(unexposedDir),
      destdir: mkdir(file(exposedDir)),
      includesfile: file('CoreExposed.includes')
    )
  }
}

// Task to merge the exposed and unexposed classes
task mergeExposed(group: 'Custom', type:Copy, dependsOn: expose) {
  description = 'Copy exposed Java types to classes.'
  // Exposed version will take precedence
  duplicatesStrategy = DuplicatesStrategy.EXCLUDE
  from file(exposedDir)
  from file(unexposedDir)
  into sourceSets.main.output.classesDirs.singleFile
}



// Attach to the classes task the placing of all compiled and exposed classes.
classes.dependsOn(mergeExposed)



// ---------------- Version-related file generation ----------------------------

/*
 * Write the information that org.python.Version reads from
 * org/python/version.properties in the class file structure. The inputs to
 * this are: information from Git (git command required); project.version;
 * and project.ext.buildDate. The task works quite hard to decode
 * project.version, which must have the correct form, to deduce whether you
 * really intend this to be a release. If anything comes after the release
 * number, typically it's a '+' sign, the version becomes a snapshot.
 *
 * ! ATTN:
 * Updating project.version will become illegal with Gradle 10, therefore
 * SNAPSHOT publications will have wrong names.
 */
task generateVersionInfo(type: WriteProperties, description: 'Write the version information as properties') {

  destinationFile = file("${processResources.destinationDir}/org/python/version.properties")
  comment = ' Jython version information (from build.gradle)'

  def projectVersion = project.version
  def buildDate = project.ext.buildDate

  // Create the properties when the task runs. But do it before the write!
  doFirst {

    /*
     * Query Git for version and tagging. The git commands are exactly
     * those used by CPython to create constants configure.ac, but do not
     * correspond very well to their names.
     */

    // Not much like a branch. Used only if git_tag is blank.
    def branch = 'git name-rev --name-only HEAD'.execute().text.split('\n', 2)[0]
    property('jython.build.git_branch', branch)
    println "  jython.build.git_branch = ${branch}"

    // When tagged, the result is "tags/vX.Y.Zrc9", or whatever.
    // When not, tagged it is the (short) revision number.
    def tag = 'git describe --all --always --dirty'.execute().text.split('\n', 2)[0]
    property('jython.build.git_tag', tag)
    println "  jython.build.git_tag = ${tag}"

    // Revision number (short = 7 hex digits)
    def ident = 'git rev-parse --short HEAD'.execute().text.split('\n', 2)[0]
    property('jython.build.git_version', ident)
    println "  jython.build.git_version = ${ident}"

    /*
     * Decompose the version string into elements for Jython to access as
     * properties. (The Ant build.xml requires them to be set in parts, but
     * we can work it out from project.version.)
     */
    // <major>.<minor>(.<micro>)(<release><serial>)?(+|-<snapshot>)?
    def versionRegex = /(\d+)\.(\d+)(\.(\d+))?((a|b|rc)(\d+))?(\+|-(\w+))?/
    def versionMatcher = projectVersion =~ versionRegex
    if (versionMatcher.count != 1) {
      throw new IllegalArgumentException("Cannot parse project version string '${project.version}'")
    }
    // In principle it could match more than once: take the leftmost
    def versionResult = versionMatcher[0]

    // <major>.<minor> means <major>.<minor>.0
    String major = versionResult[1]
    String minor = versionResult[2]
    String micro = versionResult[3] ? versionResult[4] : '0'

    // Convert the optional <release><serial> to numbers
    int level = 0, serial = 0
    if (versionResult[5]) {
      // This is some kind of pre-final release (unless snapshot)
      serial = versionResult[7] as int
      switch (versionResult[6]) {
        case 'a': level = 0xa; break // ALPHA release
        case 'b': level = 0xb; break // BETA release
        case 'rc': level = 0xc; break // release candidate
      }
    } else {
      // Not marked as a/b/rc so ostensibly a final release.
      level = 0xf
    }

    // Convert optional +|-<word> to -DEV or -SNAPSHOT suffix or empty string
    String snapshotSuffix = versionResult[8];
    if (snapshotSuffix == '+') {
      snapshotSuffix = "-SNAPSHOT"
    }

    /*
     * Work out if it looks like a release, or adjust project.version. This logic prevents us
     * releasing from a polluted repo (similar to logic in the Ant build.xml).
     */
    def L = [0:'', 10:'a', 11:'b', 12:'rc', 15:'']
    String release = "$major.$minor.$micro${L[level]}${serial?:''}"

    if (snapshotSuffix == null) {
      // The version is named correctly for a release. Make safety checks on the repo.
      String expectTag = "tags/v$release"
      String message = null;
      if (tag.endsWith('-dirty')) {
        message = 'Version-controlled files have been edited since the last commit'
      } else if (tag != expectTag) {
        message = "Change set $ident is not tagged $expectTag."
      } else {
        // Query Git for status: non-empty if uncontrolled (unignored) files.
        String gitStatus = 'git status --porcelain'.execute().text
        if (gitStatus.trim().length() > 0) {
          message = 'Workspace contains uncontrolled files'
        }
      }
      // If a message was set for any reason, fall back to a snapshot.
      if (message == null) {
        // Repository state is good for a full build.
        snapshotSuffix = ''
      } else {
        // Some reason found not to build the release.
        println "$message - build is a snapshot."
        snapshotSuffix = '-SNAPSHOT'
      }
    }

    // Rebuild the version with the snapshot suffix, even if not given originally.
    def actualProjectVersion = release + snapshotSuffix
    println "This build is for v${actualProjectVersion}."

    property('jython.version', actualProjectVersion)
    property('jython.major_version', major)
    property('jython.minor_version', minor)
    property('jython.micro_version', micro)
    property('jython.release_level', level)
    property('jython.release_serial', serial)

    /*
     * Time-stamp the build. In the time part, the ':' gets escaped to '\:', consistent with Properties.store(), unlike the Ant build.
     */
    property('jython.build.time', (new SimpleDateFormat('HH:mm:ss')).format(buildDate))
    property('jython.build.date', (new SimpleDateFormat('MMM d yyyy')).format(buildDate))
  }
}

// Attach this task to processResources
processResources.dependsOn(generateVersionInfo)






// ---------------- Copy Python Library ----------------------------------------

/*
 * The default behaviour of the Java plug-in is to make a JAR of the classes in
 * the "main" source set. We need a more complex assembly that provides users
 * with exposed classes instead of their plain counterparts, and also various
 * configuration files and the Python library.
 *
 * These copies include the tests, so we can test things :), but a subsequent
 * JarTask of the build should exclude them as necessary. (Not yet implemented.)
 */


interface Injected {
  @Inject FileSystemOperations getFs()
}

/*
 * Copy the Python standard library. We take this from a distribution of
 * CPython, but take only the files specified in CPythonLib.includes.
 * The Jython version of the standard library will be copied to the same place.
 * Files from the Jython library having the same name (relative path) as one
 * in CPythonLib.includes thereby take precedence.
 */
task mergePythonLib(type: Copy, description: 'Merge lib-python and Jython Lib') {

  def injected = project.objects.newInstance(Injected)

  // There might be a way using a collection of File rather than actual copy.
  into pythonLibDir

  // Copy Jython Lib, with precedence over CPython files of the same name
  duplicatesStrategy = DuplicatesStrategy.INCLUDE
  from libJython
  exclude '**/*.class'

  // Allow Gradle to infer the need to regenerate the outputs
  inputs.dir libJython
  inputs.dir libPython
  inputs.file file('CPythonLib.includes')

  doFirst {
    // Select the CPython stdlib files by making a list.
    def cPythonLibIncludes = []
    // Read list from CPythonLib.includes, stripping comments and spaces.
    file('CPythonLib.includes').eachLine { line ->
      def trimmed = line.split('#', 2)[0].trim()
      if (trimmed.length() > 0) {
        cPythonLibIncludes << trimmed
      }
    }
    // Copy the subset as specified by the list
    injected.fs.copy {
      into pythonLibDir
      duplicatesStrategy = DuplicatesStrategy.EXCLUDE
      from libPython
      include cPythonLibIncludes
      exclude '**/*.pyc', '**/*.pyd'
    }
  }
}

/*
 * Copy from the merge location into the main resources, excluding material
 * only needed for tests.
 */
task copyLib(type: Copy, dependsOn: mergePythonLib, description: 'Copy merged Python library (main excluding tests)') {
  into buildLibDir
  from pythonLibDir
  exclude '**/*.pyc', '**/*.pyd', '**/*.class'
  // Exclude tests and test material
  exclude libTestSpecs
}

// Attach this task to processResources
processResources.dependsOn(copyLib)

/*
 * Copy from the merge location into the test resopurces, including only
 * that extra material needed for tests.
 */
task copyTestLib(type: Copy, dependsOn: mergePythonLib, description: 'Copy merged Python library (tests only)') {
  into buildTestLibDir
  from pythonLibDir
  exclude '**/*.pyd', '**/*.class' // test material includes .pyc files
  // Include only tests and test material
  include libTestSpecs
}

// Attach this task to processResources
processTestResources.dependsOn(copyTestLib)




// ---------------- Jython-Compile Python --------------------------------------

/*
 * Compile the Python modules to .class files for the JAR. Whereas Jython runs
 * happily with a concrete Lib folder, creating and caching the .class files,
 * when Jython is supplied as a JAR, we prefer to compile the class files once
 * in advance.
 */

configurations {
  pycompile.extendsFrom(implementation)
}

dependencies {
  // Jython as built so far should be on the path of the jycompile (Ant) task
  pycompile files("$buildDir/classes/java/main")
  pycompile files("$buildDir/resources/main")
}

// A (Gradle) task to run the Ant task 'jycompile' (not pycompile).
task pycompile(group: 'Custom', description: 'Compile the Python modules to .class files for the JAR') {

  // Compiler depends on rest of Jython being fully assembled in 'classes'
  dependsOn classes
  // Note that classes depends on processResources (Java plug-in).

  // Allow Gradle to infer the need to regenerate the outputs
  inputs.dir project.ext.buildLibDir
  outputs.dir project.ext.compiledLibDir

  def inputsDir = project.ext.buildLibDir
  def outputsDir = project.ext.compiledLibDir

  doFirst {
    /*
     * Define an Ant task called 'jycompile' in the project's AntBuilder.
     * We can't define it until JythoncAntTask has been compiled, so this
     * must happen during the execution of the task (early).
     */
     ant.taskdef(
       name: 'jycompile',
       classname: 'org.python.util.JycompileAntTask',
       classpath: configurations.pycompile.asPath
     )
  }

  doLast {
    /*
     * Now use the 'jycompile' Ant task to compile the Python source we
     * supply to users. The exclusions have been copied from build.xml,
     * and also this comment:
      <!-- One might think that the exclusion of lib2to3/tests/** is
         recursive, but one would be wrong ;) It's actually only
         two levels, so for now the workaround is also to include
         data/myfixes/**

         This exclusion for lib2to3/tests/ in general is necessary
         because data/infinite_recursion.py is not compileable by
         Jython - it's too large and will exceed Java method
         limits for the top level script; nor is
         data/py3_test_grammar.py - it's Python 3. Meanwhile
         refactor.get_all_fix_names depends on having *.py, not
         exclusively $py.class, files available in
         lib2to3/tests/data/myfixes/**. -->
     */
    def exclusions = ['test/**', 'lib2to3/tests/**', 'lib2to3/tests/data/myfixes/**']
    ant.jycompile(
      srcdir: inputsDir,
      destdir: outputsDir,
      excludes: exclusions.join(',')  // Yes, it's that way round :o
    )
  }
}




// ---------------- Building the JARs ------------------------------------------

/*
 * The default behaviour of the Java plug-in is to make a JAR of the classes in
 * the "main" source set and its resources. Having carefully substituted/added
 * exposed classes in the assembled classes directory, and having prepared the
 * (compiled) stdlib as a resource, this is close to what we need, with a few
 * adjustments as noted.
 */
jar {

  // Ensure that compiled stdlib is part of the resources to JAR.
  dependsOn pycompile

  // It is important for import that X$py.class be newer than X.py
  preserveFileTimestamps = true

  // We don't JAR the expose tool itself
  exclude 'org/python/expose/generate/**'

  // Build a custom manifest
  manifest {
    // These attribute values are based on inspecting the ant build
    attributes ([
      'Main-Class': 'org.python.util.jython',
      'Built-By': 'build.gradle',
      'Automatic-Module-Name': 'org.python.jython.bison'
    ])

    attributes( [ // Build-Info section
      'version': project.version,
      'build-compiler': 'modern',
      'jdk-target-version': java.targetCompatibility,
      'debug': compileJava.options.debug,
      'informix': false,
      'oracle': false
    ], 'Build-Info' )
  }
}




tasks.withType(Javadoc) {
  // when activating doclint, make sure there are no javadoc errors (there are too many warnings)
  options.addStringOption('Xdoclint:none', '-quiet')
  mustRunAfter(pycompile)
}





// ---------------- Publication ------------------------------------------------

/*
 * Post the JAR we built to a public repository. We provide secondary -source
 * and -javadoc JARs too (supporting 'main').
 *
 * How do we test the artifact actually published is correct? The 'test' task
 * tests Jython laid out in the build directory, not the JAR we propose to
 * distribute.
 *
 * Maybe have a second JAR that contains the additional material necessary to
 * run integration tests (regression tests and others).
 */
publishing {

  publications {
    // The production JAR we expect to be cited as a dependency by users
    main(MavenPublication) {

      from components.java

      // version evaluation takes place in the configuration phase - way before generateVersionInfo runs

      pom {
        // Same description as in ~/maven/pom-template
        name = 'Jython'
        description =
          'Jython is an implementation of the high-level, dynamic, object-oriented\n' +
          'language Python written in 100% Pure Java, and seamlessly integrated with\n' +
          'the Java platform. It thus allows you to run Python on any Java platform.'
        url = 'https://www.jython.org/'
        inceptionYear = '1997'

        organization {
          name = 'Python Software Foundation'
          url = 'https://www.python.org'
        }

        // We use the PSF 2.0, but only most recently, and actually a bundle.
        licenses {
          license {
            name = 'Jython Software License'
            // Link to a stable version (defined by release tag in repo).
            url = 'https://raw.githubusercontent.com/jython/jython/v' + project.version + '/LICENSE.txt'
            distribution = 'repo'
          }
        }

        // Could list us all, but why not just the list for now?
        developers {
          developer {
            id = 'jython'
            name = 'Jython Developers'
            email = 'jython-dev@lists.sourceforge.net'
          }
        }

        mailingLists {
          mailingList {
            name = 'Jython Users'
            post = 'jython-users@lists.sourceforge.net'
          }
          mailingList {
            name = 'Jython Developers'
            post = 'jython-dev@lists.sourceforge.net'
          }
        }

        // Point to Git repositories hosted on GitHub.
        scm {
          connection = 'scm:git:https://github.com/jython/jython.git'
          developerConnection = 'scm:git:git@github.com:jython/jython.git'
          url = 'https://github.com/jython/jython'
        }

        issueManagement {
          system = 'GitHub'
          url = 'https://github.com/jython/jython/issues'
        }
      }
    }
  }

  repositories {
    // Staging area where ant -f maven/build.xml will look.
    maven {
      name = 'stagingRepo'
      url = layout.buildDirectory.dir("stagingRepo").get().asFile.toURI()
    }
  }
}

// Ensure version computation/branding precedes any publication we use.
publish.dependsOn(generateVersionInfo)




// ---------------- Java unit tests --------------------------------------------

ext {
  //distDir = relativePath("$buildDir/assembly")
  testSourceDir = relativePath('tests/java')
}


dependencies {
  // Put the exposed classes on the path of the test tasks
  testImplementation files(expose)
}

// Ensure exposed classes are ahead of standard path
sourceSets.test {
  compileClasspath  = files(expose.outputs) + compileClasspath
  runtimeClasspath = files(expose.outputs) + runtimeClasspath
  // println "runtimeClasspath = ${runtimeClasspath.asPath}"
}

compileTestJava {
  dependsOn expose
  dependsOn pycompile
  options.debug = project.compileJava.options.debug
  options.compilerArgs = ['-Xlint:deprecation', '-Xlint:unchecked']
}

test {

  dependsOn copyLib

  // Stop on first test failure
  failFast = true

  // Properties as defined in Ant target javatest-basepath
  // XXX Not sure of all that python.home is used for in tests.
  systemProperty 'python.home', file(copyLib.destinationDir).parent
  systemProperty 'python.test.source.dir', project.ext.testSourceDir
  // Place cache outside the targets for jar task
  systemProperty 'python.cachedir', "${project.buildDir}/cachedir"
  // Logging level: default is message=INFO
  //systemProperty 'python.verbose', 'CONFIG'

  include '**/*Test*'

  // Exclude based on Ant target javatest-basepath
  exclude '**/InterpTestCase'
  exclude '**/jythonTest*'            // Must run interactively
  exclude 'org/python/antlr/**'
  exclude 'org/python/tests/imp/**'   // See build.xml:importest

  // Some additional exclusions or else the task fails

  // FIXME: leaves stdin/out/err as PyFileWriter that has no fileno()
  // causing _ioTest to fail.
  exclude '**/jsr223/*'

  // FIXME: Tests that hard-code directory paths (use a symbol):
  exclude 'org/python/compiler/custom_proxymaker/**'
  exclude 'org/python/compiler/JavaMakerSmokeTest.class'

  // FIXME: Failing test finds project root from test class location
  exclude 'org/python/core/PySystemState_registry_Test.class'

  // FIXME: Fails as sys._jy_console not set when run under Gradle
  exclude 'org/python/util/InterpreterTest.class'

  doFirst {
    println "systemProperties = $systemProperties"
  }

}