diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..924837f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,81 @@ +name: CI + +on: + pull_request: + paths: + - '**.scala' + - '**.java' + - '**.sbt' + - '.scalafmt.conf' + - '.github/workflows/test.yml' + - 'project/build.properties' + push: + branches: + - master + paths: + - '**.scala' + - '**.java' + - '**.sbt' + - '.scalafmt.conf' + - '.github/workflows/test.yml' + - 'project/build.properties' + +jobs: + code_format: + name: Code format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: scalafmt + run: ./sbt scalafmtCheckAll + test_2_12: + name: Scala 2.12 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Scala 2.12 test + run: ./sbt "++2.12; test" + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: '**/target/test-reports/TEST-*.xml' + check_name: Test Report Scala 2.12 + test_2_13: + name: Scala 2.13 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Scala 2.13 test + run: ./sbt "++2.13; test" + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: '**/target/test-reports/TEST-*.xml' + check_name: Test Report Scala 2.13 + test_3: + name: Scala 3 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Scala 3 test + run: ./sbt "++3; test" + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: '**/target/test-reports/TEST-*.xml' + check_name: Test Report Scala 3 diff --git a/.gitignore b/.gitignore index 9f91569..7513e03 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ larray/src/test/resources/log-test.properties # IntelliJ specific .idea* + +.bsp \ No newline at end of file diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..b457cd5 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,7 @@ +version = 3.6.1 +project.layout = StandardConvention +runner.dialect = scala213source3 +maxColumn = 120 +style = defaultWithAlign +optIn.breaksInsideChains = true +docstrings.blankFirstLine = yes diff --git a/build.sbt b/build.sbt index 1e8ba7a..7880979 100644 --- a/build.sbt +++ b/build.sbt @@ -1,159 +1,134 @@ -sonatypeProfileName := "org.xerial" +import xerial.sbt.Sonatype._ + +Global / onChangedBuildSource := ReloadOnSourceChanges -import ReleaseTransformations._ +sonatypeProfileName := "org.xerial" -val SCALA_VERSION = "2.12.4" -val CROSS_SCALA_VERSIONS = Seq(SCALA_VERSION, "2.11.11") -scalaVersion in ThisBuild := SCALA_VERSION +val CROSS_SCALA_VERSIONS = Seq("2.12.17", "2.13.10", "3.2.1") +ThisBuild / scalaVersion := "3.2.1" -val buildSettings = Defaults.coreDefaultSettings ++ Seq( - organization := "org.xerial.larray", - organizationName := "xerial.org", - organizationHomepage := Some(new URL("http://xerial.org")), - publishMavenStyle := true, - publishArtifact in Test := false, - pomIncludeRepository := { _ => false }, - scalaVersion := SCALA_VERSION, - crossScalaVersions := CROSS_SCALA_VERSIONS, - logBuffered in Test := false, - parallelExecution := true, - parallelExecution in Test := false, - javacOptions in Compile ++= Seq("-Xlint:unchecked"), - javacOptions in(Compile, doc) := Seq( - "-locale", "en_US", - "-sourcepath", baseDirectory.value.getAbsolutePath, - "-doctitle", s"LArray ${version.value} API" +val buildSettings = Seq( + organization := "org.xerial.larray", + organizationName := "xerial.org", + publishMavenStyle := true, + Test / publishArtifact := false, + crossScalaVersions := CROSS_SCALA_VERSIONS, + Test / logBuffered := false, + parallelExecution := true, + Test / parallelExecution := false, + Compile / javacOptions ++= Seq("-Xlint:unchecked"), + Compile / doc / javacOptions := Seq( + "-locale", + "en_US", + "-sourcepath", + baseDirectory.value.getAbsolutePath, + "-doctitle", + s"LArray ${version.value} API" ), scalacOptions ++= Seq("-encoding", "UTF-8", "-unchecked", "-deprecation", "-feature"), - scalacOptions in(Compile, doc) ++= Seq("-sourcepath", baseDirectory.value.getAbsolutePath, - "-doc-source-url", "https://github.com/xerial/larray/tree/develop/€{FILE_PATH}.scala", - "-doc-title", "LArray API", - "-doc-version", version.value, + Compile / doc / scalacOptions ++= Seq( + "-sourcepath", + baseDirectory.value.getAbsolutePath, + "-doc-source-url", + "https://github.com/xerial/larray/tree/develop/€{FILE_PATH}.scala", + "-doc-title", + "LArray API", + "-doc-version", + version.value, "-diagrams" ), - testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, "-u", s"${target.value / "test-reports"}", "-o"), + libraryDependencies ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major <= 12 => + Seq() + case _ => + Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4") + } + } ++ + Seq( + "org.scala-lang.modules" %% "scala-collection-compat" % "2.8.1", + "org.wvlet.airframe" %% "airspec" % "22.11.4" % "test" + ), + testFrameworks += new TestFramework("wvlet.airspec.Framework"), crossPaths := true, licenses += ("Apache-2.0", url("https://www.apache.org/licenses/LICENSE-2.0.html")), - homepage := Some(url("https://github.com/xerial/larray")), - pomExtra := { - - scm:git:github.com/xerial/larray.git - scm:git:git@github.com:xerial/larray.git - github.com/xerial/larray.git - - - - {SCALA_VERSION} - - UTF-8 - - - - leo - Taro L. Saito - http://xerial.org/leo - - - }, - releaseTagName := { (version in ThisBuild).value }, - releaseCrossBuild := true, - releaseProcess := Seq[ReleaseStep]( - checkSnapshotDependencies, - inquireVersions, - runClean, - runTest, - setReleaseVersion, - commitReleaseVersion, - tagRelease, - ReleaseStep(action = Command.process("publishSigned", _), enableCrossBuild = true), - setNextVersion, - commitNextVersion, - ReleaseStep(action = Command.process("sonatypeReleaseAll", _), enableCrossBuild = true), - pushChanges - ), - publishTo := Some( - if (isSnapshot.value) - Opts.resolver.sonatypeSnapshots - else - Opts.resolver.sonatypeStaging - ) + sonatypeProjectHosting := Some(GitHubHosting("xerial", "larray", "leo@xerial.org")), + publishTo := sonatypePublishToBundle.value ) -lazy val root = Project( - id = "larray-root", - base = file("."), - settings = buildSettings ++ Seq( - publish := {}, - publishLocal := {}, +lazy val root = project + .in(file(".")) + .settings(buildSettings) + .settings( + name := "larray-root", + publish := {}, + publishLocal := {}, publishArtifact := false ) -) aggregate(larrayScala, larrayBuffer, larrayMMap) + .aggregate(larrayScala, larrayBuffer, larrayMMap) -val snappy = "org.xerial.snappy" % "snappy-java" % "1.1.4" -val junit = "junit" % "junit" % "4.11" % "test" -val slf4j = "org.slf4j" % "slf4j-api" % "1.7.25" -val slf4jSimple = "org.slf4j" % "slf4j-simple" % "1.7.25" +val snappy = "org.xerial.snappy" % "snappy-java" % "1.1.4" +val junit = "junit" % "junit" % "4.11" % "test" +val slf4j = "org.slf4j" % "slf4j-api" % "1.7.25" +val slf4jSimple = "org.slf4j" % "slf4j-simple" % "1.7.25" val scope = "test->test;compile->compile" -lazy val larrayScala = Project( - id = "larray", - base = file("larray"), - settings = buildSettings ++ SbtMultiJvm.multiJvmSettings ++ - Seq( - description := "LArray: A Large off-heap arrays for Scala/Java", - logBuffered in MultiJvm := false, - jvmOptions in MultiJvm ++= Seq("-Xmx128M"), - compile in MultiJvm := {(compile in MultiJvm) triggeredBy (compile in Test)}.value, - executeTests in Test := { - val testResults: Tests.Output = (executeTests in Test).value - val multiJvmTestResults: Tests.Output = (executeTests in MultiJvm).value - val results = testResults.events ++ multiJvmTestResults.events +lazy val larrayScala = + project + .in(file("larray")) + .settings(buildSettings) + .enablePlugins(MultiJvmPlugin) + .configs(MultiJvm) + .settings( + name := "larray", + description := "LArray: A Large off-heap arrays for Scala/Java", + MultiJvm / logBuffered := false, + MultiJvm / jvmOptions ++= Seq("-Xmx128M"), + MultiJvm / compile := { (MultiJvm / compile) triggeredBy (Test / compile) }.value, + Test / executeTests := { + val testResults: Tests.Output = (Test / executeTests).value + val multiJvmTestResults: Tests.Output = (MultiJvm / executeTests).value + val results = testResults.events ++ multiJvmTestResults.events Tests.Output( Tests.overall(Seq(testResults.overall, multiJvmTestResults.overall)), results, - testResults.summaries ++ multiJvmTestResults.summaries) + testResults.summaries ++ multiJvmTestResults.summaries + ) }, libraryDependencies ++= Seq( // Add dependent jars here - "org.wvlet" %% "wvlet-log" % "1.1", - snappy % "test", + "org.wvlet.airframe" %% "airframe-log" % "22.11.4", + snappy % "test", junit, - "org.iq80.snappy" % "snappy" % "0.3" % "test", - "com.novocode" % "junit-interface" % "0.11" % "test", - "org.scalatest" %% "scalatest" % "3.0.1" % "test", - "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", - "com.typesafe.akka" %% "akka-testkit" % "[2.3.14, 2.5)" % "test", - "com.typesafe.akka" %% "akka-multi-node-testkit" % "[2.3.14, 2.5)" % "test" + "org.iq80.snappy" % "snappy" % "0.3" % "test", + "com.github.sbt" % "junit-interface" % "0.13.3" % "test", + "org.scalatest" %% "scalatest" % "3.2.14" % "test", + "org.scalacheck" %% "scalacheck" % "1.15.4" % "test", + "com.typesafe.akka" %% "akka-testkit" % "2.7.0" % "test", + "com.typesafe.akka" %% "akka-multi-node-testkit" % "2.7.0" % "test" ) - ) -) dependsOn(larrayBuffer % scope, larrayMMap) configs (MultiJvm) + ).dependsOn(larrayBuffer % scope) -lazy val larrayBuffer = Project( - id = "larray-buffer", - base = file("larray-buffer"), - settings = buildSettings ++ Seq( - description := "LArray off-heap buffer library", - crossPaths := false, - autoScalaLibrary := false, - libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % "3.0.1" % "test", - "org.wvlet" %% "wvlet-log" % "1.1" % "test" - ) +lazy val larrayBuffer = project + .in(file("larray-buffer")) + .settings(buildSettings) + .settings( + name := "larray-buffer", + description := "LArray off-heap buffer library", + crossPaths := false, + autoScalaLibrary := false ) -) -lazy val larrayMMap = Project( - id = "larray-mmap", - base = file("larray-mmap"), - settings = buildSettings ++ - Seq( - description := "LArray mmap implementation", - crossPaths := false, - autoScalaLibrary := false, - libraryDependencies ++= Seq( - snappy % "test", - junit - ) +lazy val larrayMMap = project + .in(file("larray-mmap")) + .settings(buildSettings) + .settings( + description := "LArray mmap implementation", + crossPaths := false, + autoScalaLibrary := false, + libraryDependencies ++= Seq( + snappy % "test", + junit ) -) dependsOn (larrayBuffer % scope) + ).dependsOn(larrayBuffer % scope) diff --git a/larray-buffer/src/test/scala/xerial/larray/DataUnit.scala b/larray-buffer/src/test/scala/xerial/larray/DataUnit.scala index 028a229..fb6a8f6 100644 --- a/larray-buffer/src/test/scala/xerial/larray/DataUnit.scala +++ b/larray-buffer/src/test/scala/xerial/larray/DataUnit.scala @@ -24,33 +24,36 @@ package xerial.larray /** * Translators of data sizes * - * @author Taro L. Saito + * @author + * Taro L. Saito */ object DataUnit { /** * Convert the given byte size into human readable format like 1024 -> 1K * @param byteSize - * @return string representation of the byte size + * @return + * string representation of the byte size */ - def toHumanReadableFormat(byteSize:Long) : String = { + def toHumanReadableFormat(byteSize: Long): String = { // kilo, mega, giga, tera, peta, exa, zetta, yotta val unitName = Seq("", "K", "M", "G", "T", "P", "E", "Z", "Y") - def loop(index:Int, v:Long) : (Long, String) = { - if(index >= unitName.length) + def loop(index: Int, v: Long): (Long, String) = { + if (index >= unitName.length) (byteSize, "") val next = v >> 10L - if(next == 0L) + if (next == 0L) (v, unitName(index)) else - loop(index+1, next) + loop(index + 1, next) } - val (prefix, unit) = if(byteSize > 0) - loop(0, byteSize) - else - loop(0, -byteSize) match { case (p, u) => (-p, u)} + val (prefix, unit) = + if (byteSize > 0) + loop(0, byteSize) + else + loop(0, -byteSize) match { case (p, u) => (-p, u) } s"$prefix$unit" } diff --git a/larray-buffer/src/test/scala/xerial/larray/LArraySpec.scala b/larray-buffer/src/test/scala/xerial/larray/LArraySpec.scala index 52179f0..9a606b3 100644 --- a/larray-buffer/src/test/scala/xerial/larray/LArraySpec.scala +++ b/larray-buffer/src/test/scala/xerial/larray/LArraySpec.scala @@ -1,38 +1,10 @@ package xerial.larray -import org.scalatest._ -import wvlet.log.LogFormatter.SourceCodeLogFormatter -import wvlet.log.{LogSupport, Logger} -import wvlet.log.io.{ResourceReader, StopWatch, Timer} - -import scala.language.implicitConversions +import wvlet.airspec.AirSpec +import wvlet.log.io.{ResourceReader, Timer} /** - * @author Taro L. Saito - */ -trait LArraySpec - extends WordSpec - with Matchers - with ResourceReader - with Timer - with LogSupport - with BeforeAndAfterAll - with BeforeAndAfter - with BeforeAndAfterEach - with GivenWhenThen { - - implicit def toTag(t:String) = Tag(t) - - Logger.setDefaultFormatter(SourceCodeLogFormatter) - - override protected def beforeAll(): Unit = { - // Run LogLevel scanner (log-test.properties or log.properties in classpath) every 1 minute - Logger.scheduleLogLevelScan - super.beforeAll() - } - - override protected def afterAll(): Unit = { - Logger.stopScheduledLogLevelScan - super.afterAll() - } -} + * @author + * Taro L. Saito + */ +trait LArraySpec extends AirSpec with ResourceReader with Timer {} diff --git a/larray-buffer/src/test/scala/xerial/larray/buffer/LBufferTest.scala b/larray-buffer/src/test/scala/xerial/larray/buffer/LBufferTest.scala index 21daafb..82657a9 100644 --- a/larray-buffer/src/test/scala/xerial/larray/buffer/LBufferTest.scala +++ b/larray-buffer/src/test/scala/xerial/larray/buffer/LBufferTest.scala @@ -12,20 +12,20 @@ import java.nio.ByteBuffer import xerial.larray.{DataUnit, LArraySpec} /** - * @author Taro L. Saito - */ + * @author + * Taro L. Saito + */ class LBufferTest extends LArraySpec { - implicit class RichArray(m:LBuffer) { + implicit class RichArray(m: LBuffer) { def toCSV = m.toArray.mkString(", ") } + test("LBuffer") { - "LBuffer" should { - - "allocate memory" in { + test("allocate memory") { val size = 1000 - val m = new LBuffer(size) + val m = new LBuffer(size) m.putInt(0, 0) m.putInt(4, 1) m.putInt(8, 130) @@ -37,19 +37,19 @@ class LBufferTest extends LArraySpec { m.size() shouldBe size.toLong (0 until size).foreach(i => m.putByte(i, (i % 128).toByte)) - (0 until size).forall(i => m.getByte(i) == (i % 128).toByte) should be (true) + (0 until size).forall(i => m.getByte(i) == (i % 128).toByte) shouldBe (true) m.clear() - (0 until size).forall(i => m.getByte(i) == 0) should be (true) + (0 until size).forall(i => m.getByte(i) == 0) shouldBe (true) m.release() } - "convert to array" in { + test("convert to array") { val size = 12 - val m = new LBuffer(size); - for(i <- 0 until size) + val m = new LBuffer(size); + for (i <- 0 until size) m(i) = i.toByte debug(m.toCSV) @@ -57,25 +57,24 @@ class LBufferTest extends LArraySpec { debug(m.toCSV) } + test("allocate in single-thread") { - "allocate in single-thread" taggedAs("bench-single") in { - - val N = 100 + val N = 100 def range = (0 until N) - val R = 2 - val S = 1024 * 1024 + val R = 2 + val S = 1024 * 1024 info("start buffer allocation test") - time("single-thread allocation", repeat=10, blockRepeat = R) { + time("single-thread allocation", repeat = 10, blockRepeat = R) { block("without zero-filling") { - for(i <- range) yield { + for (i <- range) yield { new LBuffer(S) } } block("with zero-filling") { - for(i <- range) yield { + for (i <- range) yield { val m = new LBuffer(S) m.clear() m @@ -83,19 +82,19 @@ class LBufferTest extends LArraySpec { } block("java array") { - for(i <- range) yield { + for (i <- range) yield { new Array[Byte](S) } } block("byte buffer") { - for(i <- range) yield { + for (i <- range) yield { ByteBuffer.allocate(S) } } block("direct byte buffer") { - for(i <- range) yield { + for (i <- range) yield { ByteBuffer.allocateDirect(S) } } @@ -103,24 +102,25 @@ class LBufferTest extends LArraySpec { } } - "allocate concurrently" taggedAs("bench") in { + test("allocate concurrently") { val N = 100 - def range = (0 until N).par - val R = 2 - val S = 1024 * 1024 + + def range = collection.parallel.immutable.ParRange(0, N, 1, inclusive = false) + val R = 2 + val S = 1024 * 1024 info("start buffer allocation test") - time("concurrent allocation", repeat=10, blockRepeat = R) { + time("concurrent allocation", repeat = 10, blockRepeat = R) { block("without zero-filling") { - for(i <- range) yield { + for (i <- range) yield { new LBuffer(S) } } block("with zero-filling") { - for(i <- range) yield { + for (i <- range) yield { val m = new LBuffer(S) m.clear() m @@ -128,19 +128,19 @@ class LBufferTest extends LArraySpec { } block("java array") { - for(i <- range) yield { + for (i <- range) yield { new Array[Byte](S) } } block("byte buffer") { - for(i <- range) yield { + for (i <- range) yield { ByteBuffer.allocate(S) } } block("direct byte buffer") { - for(i <- range) yield { + for (i <- range) yield { ByteBuffer.allocateDirect(S) } } @@ -148,37 +148,36 @@ class LBufferTest extends LArraySpec { } } - "Use less memory" taggedAs("heap") in { - - // Need to produce meaningful memory usage - pending + test("Use less memory") { + pending("Need to produce meaningful memory usage") val N = 100000 val M = 1024 val rt = Runtime.getRuntime - case class Report(tag:String, free:Long, offHeap:Long, total:Long) { - override def toString = s"[${tag}] free:${DataUnit.toHumanReadableFormat(free)}, offheap:${DataUnit.toHumanReadableFormat(offHeap)}" + case class Report(tag: String, free: Long, offHeap: Long, total: Long) { + override def toString = + s"[${tag}] free:${DataUnit.toHumanReadableFormat(free)}, offheap:${DataUnit.toHumanReadableFormat(offHeap)}" } val memUsage = Seq.newBuilder[Report] - def report(tag:String) = { + def report(tag: String) = { val offHeap = LBufferConfig.allocator.allocatedSize() - val rep = Report(tag, rt.freeMemory(), offHeap, rt.totalMemory()) - //memUsage += rep + val rep = Report(tag, rt.freeMemory(), offHeap, rt.totalMemory()) + // memUsage += rep rep } - var r1 : Seq[Array[Byte]] = null - var r2 : Seq[LBuffer] = null + var r1: Seq[Array[Byte]] = null + var r2: Seq[LBuffer] = null - time("memory allocation", repeat=3) { + time("memory allocation", repeat = 3) { Thread.sleep(5000) block("Array") { info(report("Array")) - val result = for(i <- 0 until N) yield { + val result = for (i <- 0 until N) yield { val a = new Array[Byte](M) report("Array") a @@ -192,7 +191,7 @@ class LBufferTest extends LArraySpec { block("LBuffer") { info(report("LBuffer")) - val result = for(i <- 0 until N) yield { + val result = for (i <- 0 until N) yield { val l = new LBuffer(M) report("LBuffer") l @@ -202,15 +201,14 @@ class LBufferTest extends LArraySpec { } - } - "read from ByteBuffer" in { - val bytes = Array[Byte](1, 2, 3) + test("read from ByteBuffer") { + val bytes = Array[Byte](1, 2, 3) val byteBuffer = ByteBuffer.wrap(bytes) - val lbuffer = new LBuffer(3) + val lbuffer = new LBuffer(3) lbuffer.readFrom(byteBuffer, 0) - byteBuffer.array() === lbuffer.toArray + byteBuffer.array() shouldBe lbuffer.toArray } } -} \ No newline at end of file +} diff --git a/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala b/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala index a743258..cb64bb9 100644 --- a/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala +++ b/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala @@ -24,36 +24,38 @@ package xerial.larray.buffer import xerial.larray.LArraySpec - /** - * @author Taro L. Saito - */ + * @author + * Taro L. Saito + */ class MemoryAllocatorTest extends LArraySpec { - "ConcurrentMemoryAllocator" should { - "perform better than the default heap allocator" in { + test("ConcurrentMemoryAllocator") { + test("perform better than the default heap allocator") { val N = 1000 val B = 64 * 1024 val t = time("alloc", repeat = 5) { block("concurrent") { - val l = for (i <- (0 until N).par) yield { + val range = collection.parallel.immutable.ParRange(0, N, 1, inclusive = false) + val l = for (i <- range) yield { val a = new LBuffer(B) - a(B-1) = 1.toByte + a(B - 1) = 1.toByte a } l.foreach(_.release) } block("Array") { - val l = for (i <- (0 until N).par) yield { + val range = collection.parallel.immutable.ParRange(0, N, 1, inclusive = false) + val l = for (i <- range) yield { val a = new Array[Int](B) - a(B-1) = 1 + a(B - 1) = 1 a } } } - t("concurrent") should be <= (t("Array")) + t("concurrent") < t("Array") shouldBe true } } -} \ No newline at end of file +} diff --git a/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala b/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala index 5cd8093..37b70d5 100644 --- a/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala +++ b/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala @@ -10,15 +10,16 @@ package xerial.larray.buffer import xerial.larray.LArraySpec /** - * @author Taro L. Saito - */ + * @author + * Taro L. Saito + */ class WrappedLBufferTest extends LArraySpec { - "WrappedLBuffer" should { + test("WrappedLBuffer") { - "be a subrange of LBuffer" in { + test("be a subrange of LBuffer") { val l = new LBuffer(10) - for(i <- 0 until l.size().toInt) { + for (i <- 0 until l.size().toInt) { l(i) = (10 - i).toByte } @@ -27,8 +28,8 @@ class WrappedLBufferTest extends LArraySpec { debug(v.toArray.mkString(", ")) v.size() shouldBe 8 - 3 - v.toArray.zipWithIndex.forall{case (a, i) => a == l(i+3)} + v.toArray.zipWithIndex.forall { case (a, i) => a == l(i + 3) } } } -} \ No newline at end of file +} diff --git a/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java b/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java index 4307625..3daa447 100644 --- a/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java +++ b/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java @@ -7,7 +7,6 @@ import java.io.RandomAccessFile; import java.lang.reflect.Field; import java.nio.channels.FileChannel; -import sun.misc.SharedSecrets; import xerial.larray.buffer.LBufferAPI; import xerial.larray.buffer.LBufferConfig; import xerial.larray.buffer.UnsafeUtil; @@ -95,7 +94,8 @@ public MMapBuffer(File f, long offset, long size, MMapMode mode) throws IOExcept //trace(f"mmap addr:$rawAddr%x, start address:${rawAddr+pagePosition}%x") if(OSInfo.isWindows()) { - sun.misc.JavaIOFileDescriptorAccess a = SharedSecrets.getJavaIOFileDescriptorAccess(); + // TODO SharedSecrets no longer accessible in Java 17 + sun.misc.JavaIOFileDescriptorAccess a = sun.misc.SharedSecrets.getJavaIOFileDescriptorAccess(); winHandle = LArrayNative.duplicateHandle(a.getHandle(raf.getFD())); //debug(f"win handle: $winHandle%x") } diff --git a/larray-mmap/src/test/scala/xerial/larray/impl/LArrayLoaderTest.scala b/larray-mmap/src/test/scala/xerial/larray/impl/LArrayLoaderTest.scala index 194b997..211f4b2 100644 --- a/larray-mmap/src/test/scala/xerial/larray/impl/LArrayLoaderTest.scala +++ b/larray-mmap/src/test/scala/xerial/larray/impl/LArrayLoaderTest.scala @@ -1,73 +1,74 @@ -/*-------------------------------------------------------------------------- - * Copyright 2013 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// -// LArrayLoaderTest.scala -// Since: 2013/03/29 9:56 -// -//-------------------------------------- - -package xerial.larray.impl - -import java.net.{URL, URLClassLoader} -import java.io.File -import xerial.larray.LArraySpec - - -/** - * @author Taro L. Saito - */ -class LArrayLoaderTest extends LArraySpec { - - "LArrayLoader" should { - - "use a different library name for each class loader" in { - - val larrayNativeCls = "xerial.larray.impl.LArrayNative" - - val cl = Thread.currentThread().getContextClassLoader - val parentCl = cl.getParent.getParent // Fix for sbt-0.13 - debug(s"context cl: ${cl}, parent cl: $parentCl") - val classPath = Array(new File("larray-mmap/target/classes").toURI.toURL) - val cl1 = new URLClassLoader(classPath, parentCl) - val cl2 = new URLClassLoader(classPath, parentCl) - - import java.{lang=>jl} - - val nativeCls1 = cl1.loadClass(larrayNativeCls) - val ni1 = nativeCls1.newInstance() - val arr1 = Array.ofDim[Byte](100) - val m1 = nativeCls1.getDeclaredMethod("copyToArray", jl.Long.TYPE, classOf[AnyRef], jl.Integer.TYPE, jl.Integer.TYPE) - m1.invoke(ni1, Seq.apply[AnyRef](new jl.Long(0L), arr1, new jl.Integer(0), new jl.Integer(0)):_*) - val nativeCls1_2 = cl1.loadClass(larrayNativeCls) - - val nativeCls2 = cl2.loadClass(larrayNativeCls) - val ni2 = nativeCls2.newInstance() - val arr2 = Array.ofDim[Byte](100) - val m2 = nativeCls2.getDeclaredMethod("copyToArray", jl.Long.TYPE, classOf[AnyRef], jl.Integer.TYPE, jl.Integer.TYPE) - m2.invoke(ni1, Seq.apply[AnyRef](new jl.Long(0L), arr2, new jl.Integer(0), new jl.Integer(0)):_*) - - nativeCls1 should not be (nativeCls2) - nativeCls1 should be (nativeCls1_2) - - val arr3 = Array.ofDim[Byte](100) - LArrayNative.copyToArray(0, arr3, 0, 0) - } - - - } - -} \ No newline at end of file +/*-------------------------------------------------------------------------- + * Copyright 2013 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// +// LArrayLoaderTest.scala +// Since: 2013/03/29 9:56 +// +//-------------------------------------- + +package xerial.larray.impl + +import java.net.{URL, URLClassLoader} +import java.io.File +import xerial.larray.LArraySpec + +/** + * @author + * Taro L. Saito + */ +class LArrayLoaderTest extends LArraySpec { + + "LArrayLoader" should { + + "use a different library name for each class loader" in { + + val larrayNativeCls = "xerial.larray.impl.LArrayNative" + + val cl = Thread.currentThread().getContextClassLoader + val parentCl = cl.getParent.getParent // Fix for sbt-0.13 + debug(s"context cl: ${cl}, parent cl: $parentCl") + val classPath = Array(new File("larray-mmap/target/classes").toURI.toURL) + val cl1 = new URLClassLoader(classPath, parentCl) + val cl2 = new URLClassLoader(classPath, parentCl) + + import java.{lang => jl} + + val nativeCls1 = cl1.loadClass(larrayNativeCls) + val ni1 = nativeCls1.newInstance() + val arr1 = Array.ofDim[Byte](100) + val m1 = + nativeCls1.getDeclaredMethod("copyToArray", jl.Long.TYPE, classOf[AnyRef], jl.Integer.TYPE, jl.Integer.TYPE) + m1.invoke(ni1, Seq.apply[AnyRef](new jl.Long(0L), arr1, new jl.Integer(0), new jl.Integer(0)): _*) + val nativeCls1_2 = cl1.loadClass(larrayNativeCls) + + val nativeCls2 = cl2.loadClass(larrayNativeCls) + val ni2 = nativeCls2.newInstance() + val arr2 = Array.ofDim[Byte](100) + val m2 = + nativeCls2.getDeclaredMethod("copyToArray", jl.Long.TYPE, classOf[AnyRef], jl.Integer.TYPE, jl.Integer.TYPE) + m2.invoke(ni1, Seq.apply[AnyRef](new jl.Long(0L), arr2, new jl.Integer(0), new jl.Integer(0)): _*) + + nativeCls1 should not be (nativeCls2) + nativeCls1 shouldBe (nativeCls1_2) + + val arr3 = Array.ofDim[Byte](100) + LArrayNative.copyToArray(0, arr3, 0, 0) + } + + } + +} diff --git a/larray/src/main/java/xerial/larray/japi/LArrayJ.java b/larray/src/main/java/xerial/larray/japi/LArrayJ.java index 87f40ab..4b34090 100644 --- a/larray/src/main/java/xerial/larray/japi/LArrayJ.java +++ b/larray/src/main/java/xerial/larray/japi/LArrayJ.java @@ -19,7 +19,7 @@ import xerial.larray.*; import xerial.larray.buffer.LBufferConfig; import xerial.larray.buffer.MemoryAllocator; -import xerial.larray.mmap.MMapMode; +//import xerial.larray.mmap.MMapMode; import java.io.File; @@ -31,13 +31,13 @@ public class LArrayJ { static MemoryAllocator defaultAllocator() { return LBufferConfig.allocator; } - public static MappedLByteArray mmap(File f, MMapMode mode) { - return new MappedLByteArray(f, 0L, f.length(), mode, defaultAllocator()); - } - - public static MappedLByteArray mmap(File f, long offset, long size, MMapMode mode) { - return new MappedLByteArray(f, offset, size, mode, defaultAllocator()); - } +// public static MappedLByteArray mmap(File f, MMapMode mode) { +// return new MappedLByteArray(f, 0L, f.length(), mode, defaultAllocator()); +// } +// +// public static MappedLByteArray mmap(File f, long offset, long size, MMapMode mode) { +// return new MappedLByteArray(f, offset, size, mode, defaultAllocator()); +// } public static LByteArray newLByteArray(long size) { diff --git a/larray/src/main/scala/xerial/larray/AltLArray.scala b/larray/src/main/scala/xerial/larray/AltLArray.scala index d520ade..bfcc86c 100644 --- a/larray/src/main/scala/xerial/larray/AltLArray.scala +++ b/larray/src/main/scala/xerial/larray/AltLArray.scala @@ -22,35 +22,37 @@ package xerial.larray +import xerial.larray.LArrayView.LIntArrayView + /** - * A common trait for alternative implementations of LArray. This implementation is provided only for testing purpose, so many features might be missing - * in LArrays impemented this trait. - */ + * A common trait for alternative implementations of LArray. This implementation is provided only for testing purpose, + * so many features might be missing in LArrays impemented this trait. + */ trait AltLIntArrayImpl extends LArray[Int] { def address = LArray.EmptyArray.address // throws an exception - def copyTo(dest:LByteArray, destOffset:Long) { + def copyTo(dest: LByteArray, destOffset: Long): Unit = { throw new UnsupportedOperationException("copyTo") } - def copyTo[B](srcOffset:Long, dest:RawByteArray[B], destOffset:Long, blen:Long) { + def copyTo[B](srcOffset: Long, dest: RawByteArray[B], destOffset: Long, blen: Long): Unit = { throw new UnsupportedOperationException("copyTo") } - def view(from: Long, to: Long) = new LArrayView.LIntArrayView(this, from, to - from) + def view(from: Long, to: Long): LIntArrayView = new LArrayView.LIntArrayView(this, from, to - from) } /** - * Alternative implementation of LArray that might be inefficient, but written for comparing performances. - * LIntArraySimple wraps Array[Int] to support Long-type indexes - * @param size array size - */ + * Alternative implementation of LArray that might be inefficient, but written for comparing performances. + * LIntArraySimple wraps Array[Int] to support Long-type indexes + * @param size + * array size + */ class LIntArraySimple(val size: Long) extends LArray[Int] with AltLIntArrayImpl { protected[this] def newBuilder = LArray.newBuilder[Int] - private def boundaryCheck(i: Long) { if (i > Int.MaxValue) sys.error(f"index must be smaller than ${Int.MaxValue}%,d") @@ -65,13 +67,13 @@ class LIntArraySimple(val size: Long) extends LArray[Int] with AltLIntArrayImpl } def apply(i: Long): Int = { - //boundaryCheck(i) + // boundaryCheck(i) arr.apply(i.toInt) } // a(i) = a(j) = 1 def update(i: Long, v: Int): Int = { - //boundaryCheck(i) + // boundaryCheck(i) arr.update(i.toInt, v) v } @@ -81,64 +83,66 @@ class LIntArraySimple(val size: Long) extends LArray[Int] with AltLIntArrayImpl } /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize: Int = 4 - - } - /** - * Emulate large arrays using two-diemensional matrix of Int. Array[Int](page index)(offset in page) - * @param size array size - */ -class MatrixBasedLIntArray(val size:Long) extends LArray[Int] with AltLIntArrayImpl { + * Emulate large arrays using two-diemensional matrix of Int. Array[Int](page index)(offset in page) + * @param size + * array size + */ +class MatrixBasedLIntArray(val size: Long) extends LArray[Int] with AltLIntArrayImpl { private[larray] def elementByteSize: Int = 4 protected[this] def newBuilder = LArray.newBuilder[Int] + private val maskLen: Int = 24 + private val B: Int = 1 << maskLen // block size + private val mask: Long = ~(~0L << maskLen) - private val maskLen : Int = 24 - private val B : Int = 1 << maskLen // block size - private val mask : Long = ~(~0L << maskLen) - - @inline private def index(i:Long) : Int = (i >>> maskLen).toInt - @inline private def offset(i:Long) : Int = (i & mask).toInt + @inline private def index(i: Long): Int = (i >>> maskLen).toInt + @inline private def offset(i: Long): Int = (i & mask).toInt - private val numBlocks = ((size + (B - 1L))/ B).toInt - private val arr = Array.ofDim[Int](numBlocks, B) + private val numBlocks = ((size + (B - 1L)) / B).toInt + private val arr = Array.ofDim[Int](numBlocks, B) def clear() { - for(a <- arr) { + for (a <- arr) { java.util.Arrays.fill(a, 0, a.length, 0) } } /** - * Retrieve an element - * @param i index - * @return the element value - */ + * Retrieve an element + * @param i + * index + * @return + * the element value + */ def apply(i: Long) = arr(index(i))(offset(i)) /** - * Update an element - * @param i index to be updated - * @param v value to set - * @return the value - */ + * Update an element + * @param i + * index to be updated + * @param v + * value to set + * @return + * the value + */ def update(i: Long, v: Int) = { arr(index(i))(offset(i)) = v v } /** - * Release the memory of LArray. After calling this method, the results of calling the other methods becomes undefined or might cause JVM crash. - */ + * Release the memory of LArray. After calling this method, the results of calling the other methods becomes + * undefined or might cause JVM crash. + */ def free {} - } diff --git a/larray/src/main/scala/xerial/larray/LArray.scala b/larray/src/main/scala/xerial/larray/LArray.scala index bc4d1e7..8064b09 100644 --- a/larray/src/main/scala/xerial/larray/LArray.scala +++ b/larray/src/main/scala/xerial/larray/LArray.scala @@ -31,272 +31,281 @@ import java.io.{File, FileInputStream, FileOutputStream} import wvlet.log.LogSupport import xerial.larray.buffer.{Memory, MemoryAllocator} -import xerial.larray.mmap.MMapMode - +//import xerial.larray.mmap.MMapMode /** - * Read-only interface of [[xerial.larray.LArray]] - * @tparam A - */ + * Read-only interface of [[xerial.larray.LArray]] + * @tparam A + */ trait LSeq[A] extends LIterable[A] { + /** - * Element size of this array - * @return size of this array - */ + * Element size of this array + * @return + * size of this array + */ def size: Long /** - * Byte length of this array. - * @return - */ + * Byte length of this array. + * @return + */ def byteLength: Long = elementByteSize * size /** - * Retrieve an element at the given index. LArray does not perform boundary checks for optimizing the performance, - * so reading the indexes out of bounds might cause JVM crash. - * @param i index - * @return the element value - */ + * Retrieve an element at the given index. LArray does not perform boundary checks for optimizing the performance, so + * reading the indexes out of bounds might cause JVM crash. + * @param i + * index + * @return + * the element value + */ @inline def apply(i: Long): A /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize: Int - /** - * Create a sequence of DirectByteBuffer that projects LArray contents - * @return sequence of `java.nio.ByteBuffer` - */ + * Create a sequence of DirectByteBuffer that projects LArray contents + * @return + * sequence of `java.nio.ByteBuffer` + */ def toDirectByteBuffer: Array[ByteBuffer] = { - var pos = 0L - val b = Array.newBuilder[ByteBuffer] + var pos = 0L + val b = Array.newBuilder[ByteBuffer] val limit = byteLength while (pos < limit) { val len: Long = math.min(limit - pos, Int.MaxValue) - val d = UnsafeUtil.newDirectByteBuffer(address + pos, len.toInt) + val d = UnsafeUtil.newDirectByteBuffer(address + pos, len.toInt) b += d.order(ByteOrder.nativeOrder()) pos += len } b.result() } - /** - * Save to a file. - * @param f - * @return - */ + * Save to a file. + * @param f + * @return + */ def saveTo(f: File): File = { val fout = new FileOutputStream(f).getChannel try { fout.write(this.toDirectByteBuffer) f - } - finally + } finally fout.close } /** - * Copy the contents of this LSeq[A] into the target LByteArray - * @param dst - * @param dstOffset - */ + * Copy the contents of this LSeq[A] into the target LByteArray + * @param dst + * @param dstOffset + */ def copyTo(dst: LByteArray, dstOffset: Long) /** - * Copy the contents of this sequence into the target LByteArray - * @param srcOffset - * @param dst - * @param dstOffset - * @param blen the byte length to copy - */ + * Copy the contents of this sequence into the target LByteArray + * @param srcOffset + * @param dst + * @param dstOffset + * @param blen + * the byte length to copy + */ def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) - /** - * Raw-memory address of this array - */ + * Raw-memory address of this array + */ @inline def address: Long import UnsafeUtil.unsafe - @inline def getByte(offset: Long): Byte = unsafe.getByte(address + offset) - @inline def getChar(offset: Long): Char = unsafe.getChar(address + offset) - @inline def getShort(offset: Long): Short = unsafe.getShort(address + offset) - @inline def getInt(offset: Long): Int = unsafe.getInt(address + offset) - @inline def getFloat(offset: Long): Float = unsafe.getFloat(address + offset) - @inline def getLong(offset: Long): Long = unsafe.getLong(address + offset) + @inline def getByte(offset: Long): Byte = unsafe.getByte(address + offset) + @inline def getChar(offset: Long): Char = unsafe.getChar(address + offset) + @inline def getShort(offset: Long): Short = unsafe.getShort(address + offset) + @inline def getInt(offset: Long): Int = unsafe.getInt(address + offset) + @inline def getFloat(offset: Long): Float = unsafe.getFloat(address + offset) + @inline def getLong(offset: Long): Long = unsafe.getLong(address + offset) @inline def getDouble(offset: Long): Double = unsafe.getDouble(address + offset) - } /** - * LArray is a mutable large array. - * - * The differences from the standard Array[A] are: - * - * - LArray accepts Long type indexes, so it is possible to have more than 2G (2^31-1) entries, which is a limitation of the standard Array[A]. - * - The memory resource of LArray[A] resides outside of the normal garbage-collected JVM heap and can be released via [[xerial.larray.LArray.free]]. - * - If [[LArray.free]] is not called, the acquried memory stays until LArray is collected by GC. - * - LArray elements are not initialized, so explicit initialization using [[LArray.clear]] is necessary. - * - * == Usage == - * {{{ - import xerial.larray._ - - // Create a new LArray of Int type - val l = LArray.of[Int](5) - - // Create an LArray with initial values - val ll = LArray(3, 5, 9, 10) - - // Set elements - for(i <- 0 until l.size.toInt) - l(i) = i - - // Read elements - val e0 = l(0) - val e1 = l(1) - - // Print the elements - println(l.mkString(", ")) // 0, 1, 2, 3, 4 - - // Traverse the elements with their indexes - for((e, i) <- l.zipWithIndex) - println(s"l(\$i) = \$e") // l(0) = 0, l(1) = 1, ... - - // Manipulate LArray - val l2 = l.map(_ * 10) // LArray(0, 10, 20, 30, 40) - val f = l.filter(_ % 2 == 0) // LArray(0, 2, 4) - val s = l.slice(2) // LArray(2, 3, 4) - l.foreach(println(_)) - - // Build LArray - val b = LArray.newBuilder[Int] - for(i <- 0 until (10, step=3)) - b += i - val lb = b.result // LArray(0, 3, 6, 9) - - // Convert to Scala Array - val arr = l.toArray - println(arr.mkString(", ")) // 0, 1, 2, 3, 4 - - // Convert Scala Array to LArray - val arr2 = Array(1, 3, 5) - val la = arr2.toLArray - - // Save to a file - import java.io.File - val file = l.saveTo(new File("target/larray.tmp")) - // Load from a file - val l3 = LArray.loadFrom[Int](file) // LArray(0, 1, 2, 3, 4) - - // Initialize the array - l.clear() - println(l.mkString(", ")) // 0, 0, 0, 0, 0 - - // Release the memory contents. - l.free - l3.free - - // You can omit calling free, because GC collects unused LArrays -}}} - - * - * @tparam A element type - */ + * LArray is a mutable large array. + * + * The differences from the standard Array[A] are: + * + * - LArray accepts Long type indexes, so it is possible to have more than 2G (2^31-1) entries, which is a limitation + * of the standard Array[A]. + * - The memory resource of LArray[A] resides outside of the normal garbage-collected JVM heap and can be released + * via [[xerial.larray.LArray.free]]. + * - If [[LArray.free]] is not called, the acquried memory stays until LArray is collected by GC. + * - LArray elements are not initialized, so explicit initialization using [[LArray.clear]] is necessary. + * + * ==Usage== + * {{{ + * import xerial.larray._ + * + * // Create a new LArray of Int type + * val l = LArray.of[Int](5) + * + * // Create an LArray with initial values + * val ll = LArray(3, 5, 9, 10) + * + * // Set elements + * for(i <- 0 until l.size.toInt) + * l(i) = i + * + * // Read elements + * val e0 = l(0) + * val e1 = l(1) + * + * // Print the elements + * println(l.mkString(", ")) // 0, 1, 2, 3, 4 + * + * // Traverse the elements with their indexes + * for((e, i) <- l.zipWithIndex) + * println(s"l(\$i) = \$e") // l(0) = 0, l(1) = 1, ... + * + * // Manipulate LArray + * val l2 = l.map(_ * 10) // LArray(0, 10, 20, 30, 40) + * val f = l.filter(_ % 2 == 0) // LArray(0, 2, 4) + * val s = l.slice(2) // LArray(2, 3, 4) + * l.foreach(println(_)) + * + * // Build LArray + * val b = LArray.newBuilder[Int] + * for(i <- 0 until (10, step=3)) + * b += i + * val lb = b.result // LArray(0, 3, 6, 9) + * + * // Convert to Scala Array + * val arr = l.toArray + * println(arr.mkString(", ")) // 0, 1, 2, 3, 4 + * + * // Convert Scala Array to LArray + * val arr2 = Array(1, 3, 5) + * val la = arr2.toLArray + * + * // Save to a file + * import java.io.File + * val file = l.saveTo(new File("target/larray.tmp")) + * // Load from a file + * val l3 = LArray.loadFrom[Int](file) // LArray(0, 1, 2, 3, 4) + * + * // Initialize the array + * l.clear() + * println(l.mkString(", ")) // 0, 0, 0, 0, 0 + * + * // Release the memory contents. + * l.free + * l3.free + * + * // You can omit calling free, because GC collects unused LArrays + * }}} + * + * @tparam A + * element type + */ trait LArray[A] extends LSeq[A] with WritableByteChannel { def isOpen: Boolean = true - def close() { + def close(): Unit = { free } /** - * Release the memory of LArray. After calling this method, the results of calling the other methods becomes undefined or might cause JVM crash. - */ - def free + * Release the memory of LArray. After calling this method, the results of calling the other methods becomes + * undefined or might cause JVM crash. + */ + def free: Unit /** - * Release the memory of LArray. After calling this mehtod, thr results of calling the other methods becomes undefined or might cause JVM crash. - */ - def release = free + * Release the memory of LArray. After calling this mehtod, thr results of calling the other methods becomes + * undefined or might cause JVM crash. + */ + def release: Unit = free /** - * Wraps with immutable interface - * @return - */ + * Wraps with immutable interface + * @return + */ def toLSeq: LSeq[A] = this.asInstanceOf[LSeq[A]] /** - * Clear the contents of the array. It simply fills the array with zero bytes. - */ - def clear() + * Clear the contents of the array. It simply fills the array with zero bytes. + */ + def clear(): Unit /** - * Write the contents of ByteBuffer to this array. This method increments the internal cursor. - * @param src - * @return - */ + * Write the contents of ByteBuffer to this array. This method increments the internal cursor. + * @param src + * @return + */ def write(src: ByteBuffer): Int = throw new UnsupportedOperationException("write(ByteBuffer)") - /** - * Update an element - * @param i index to be updated - * @param v value to set - * @return the value - */ + * Update an element + * @param i + * index to be updated + * @param v + * value to set + * @return + * the value + */ @inline def update(i: Long, v: A): A /** - * Create a shallow copy (view) of LArray - * @param from - * @param to - * @return - */ + * Create a shallow copy (view) of LArray + * @param from + * @param to + * @return + */ def view(from: Long, to: Long): LArrayView[A] override def toString = mkString(", ") import UnsafeUtil.unsafe - @inline def putByte(offset: Long, v: Byte) = { unsafe.putByte(address+offset, v); v } - @inline def putChar(offset: Long, v: Char) = { unsafe.putChar(address+offset, v); v } - @inline def putShort(offset: Long, v: Short) = { unsafe.putShort(address+offset, v); v } - @inline def putInt(offset: Long, v: Int) = { unsafe.putInt(address+offset, v); v } - @inline def putFloat(offset: Long, v: Float) = { unsafe.putFloat(address + offset, v); v } - @inline def putLong(offset: Long, v: Long) = { unsafe.putLong(address + offset, v); v} - @inline def putDouble(offset: Long, v: Double) = { unsafe.putDouble(address + offset, v); v } + @inline def putByte(offset: Long, v: Byte): Unit = { unsafe.putByte(address + offset, v); v } + @inline def putChar(offset: Long, v: Char): Unit = { unsafe.putChar(address + offset, v); v } + @inline def putShort(offset: Long, v: Short): Unit = { unsafe.putShort(address + offset, v); v } + @inline def putInt(offset: Long, v: Int): Unit = { unsafe.putInt(address + offset, v); v } + @inline def putFloat(offset: Long, v: Float): Unit = { unsafe.putFloat(address + offset, v); v } + @inline def putLong(offset: Long, v: Long): Unit = { unsafe.putLong(address + offset, v); v } + @inline def putDouble(offset: Long, v: Double): Unit = { unsafe.putDouble(address + offset, v); v } } - /** - * LArray factory - * - * {{{ - * - * // Create a new LArray[Int] of size 10 - * LArray.of[Int](10) - * }}} - * @author Taro L. Saito - */ + * LArray factory + * + * {{{ + * + * // Create a new LArray[Int] of size 10 + * LArray.of[Int](10) + * }}} + * @author + * Taro L. Saito + */ object LArray { import _root_.java.{lang => jl} /** - * Load the contents of a file into LArray - * @param f the file to read - * @tparam A the element type - * @return LArray contains the file contents - */ + * Load the contents of a file into LArray + * @param f + * the file to read + * @tparam A + * the element type + * @return + * LArray contains the file contents + */ def loadFrom[A: ClassTag](f: File): LArray[A] = { val fin = new FileInputStream(f).getChannel val tag = implicitly[ClassTag[A]] @@ -304,12 +313,12 @@ object LArray { tag.runtimeClass match { case jl.Boolean.TYPE => { val arr = new Array[Byte](8) - val bb = ByteBuffer.wrap(arr) + val bb = ByteBuffer.wrap(arr) fin.read(bb) - val numBits = bb.getLong(0) + val numBits = bb.getLong(0) val fileSize = fin.size() - val b = new LLongArrayBuilder() - var pos = 8L + val b = new LLongArrayBuilder() + var pos = 8L b.sizeHint((fileSize - pos) / 8L) while (pos < fileSize) { pos += fin.transferTo(pos, fileSize - pos, b) @@ -317,9 +326,9 @@ object LArray { new LBitArray(b.result, numBits).asInstanceOf[LArray[A]] } case _ => { - var pos = 0L + var pos = 0L val fileSize = fin.size() - val b = LArray.newBuilder[A] + val b = LArray.newBuilder[A] b.sizeHint(fileSize / b.elementSize) while (pos < fileSize) { pos += fin.transferTo(pos, fileSize - pos, b) @@ -327,20 +336,15 @@ object LArray { b.result() } } - } - finally + } finally fin.close - } - - private[larray] object EmptyArray - extends LArray[Nothing] - with LIterable[Nothing] { + private[larray] object EmptyArray extends LArray[Nothing] with LIterable[Nothing] { private[larray] def elementByteSize: Int = 0 - def clear() {} + def clear(): Unit = {} override def toDirectByteBuffer = Array.empty @@ -371,93 +375,97 @@ object LArray { def view(from: Long, to: Long) = LArrayView.EmptyView /** - * Raw-memory address of this array - */ - def address : Long = { throw new UnsupportedOperationException("address") } + * Raw-memory address of this array + */ + def address: Long = { throw new UnsupportedOperationException("address") } } - /** - * Creates a new LArray of the specified type and size - * @param size the array size - * @tparam A the element type - * @return a new instance of LArray[A] of the given size - */ + * Creates a new LArray of the specified type and size + * @param size + * the array size + * @tparam A + * the element type + * @return + * a new instance of LArray[A] of the given size + */ def of[A: ClassTag](size: Long): LArray[A] = { val tag = implicitly[ClassTag[A]] tag.runtimeClass match { - case jl.Integer.TYPE => new LIntArray(size).asInstanceOf[LArray[A]] - case jl.Byte.TYPE => new LByteArray(size).asInstanceOf[LArray[A]] + case jl.Integer.TYPE => new LIntArray(size).asInstanceOf[LArray[A]] + case jl.Byte.TYPE => new LByteArray(size).asInstanceOf[LArray[A]] case jl.Character.TYPE => new LCharArray(size).asInstanceOf[LArray[A]] - case jl.Short.TYPE => new LShortArray(size).asInstanceOf[LArray[A]] - case jl.Long.TYPE => new LLongArray(size).asInstanceOf[LArray[A]] - case jl.Float.TYPE => new LFloatArray(size).asInstanceOf[LArray[A]] - case jl.Double.TYPE => new LDoubleArray(size).asInstanceOf[LArray[A]] - case jl.Boolean.TYPE => new LBitArray(size).asInstanceOf[LArray[A]] - case _ => LObjectArray.ofDim[A](size) + case jl.Short.TYPE => new LShortArray(size).asInstanceOf[LArray[A]] + case jl.Long.TYPE => new LLongArray(size).asInstanceOf[LArray[A]] + case jl.Float.TYPE => new LFloatArray(size).asInstanceOf[LArray[A]] + case jl.Double.TYPE => new LDoubleArray(size).asInstanceOf[LArray[A]] + case jl.Boolean.TYPE => new LBitArray(size).asInstanceOf[LArray[A]] + case _ => LObjectArray.ofDim[A](size) } } val emptyBooleanArray = LArray.of[Boolean](0) - val emptyByteArray = LArray.of[Byte](0) - val emptyCharArray = LArray.of[Char](0) - val emptyDoubleArray = LArray.of[Double](0) - val emptyFloatArray = LArray.of[Float](0) - val emptyIntArray = LArray.of[Int](0) - val emptyLongArray = LArray.of[Long](0) - val emptyShortArray = LArray.of[Short](0) - val emptyObjectArray = LArray.of[Object](0) + val emptyByteArray = LArray.of[Byte](0) + val emptyCharArray = LArray.of[Char](0) + val emptyDoubleArray = LArray.of[Double](0) + val emptyFloatArray = LArray.of[Float](0) + val emptyIntArray = LArray.of[Int](0) + val emptyLongArray = LArray.of[Long](0) + val emptyShortArray = LArray.of[Short](0) + val emptyObjectArray = LArray.of[Object](0) /** - * Empty array - * @tparam A - * @return - */ + * Empty array + * @tparam A + * @return + */ def empty[A: ClassTag]: LArray[A] = { val tag = implicitly[ClassTag[A]] tag.runtimeClass match { - case jl.Integer.TYPE => emptyIntArray.asInstanceOf[LArray[A]] - case jl.Byte.TYPE => emptyByteArray.asInstanceOf[LArray[A]] + case jl.Integer.TYPE => emptyIntArray.asInstanceOf[LArray[A]] + case jl.Byte.TYPE => emptyByteArray.asInstanceOf[LArray[A]] case jl.Character.TYPE => emptyCharArray.asInstanceOf[LArray[A]] - case jl.Short.TYPE => emptyShortArray.asInstanceOf[LArray[A]] - case jl.Long.TYPE => emptyLongArray.asInstanceOf[LArray[A]] - case jl.Float.TYPE => emptyFloatArray.asInstanceOf[LArray[A]] - case jl.Double.TYPE => emptyDoubleArray.asInstanceOf[LArray[A]] - case jl.Boolean.TYPE => emptyBooleanArray.asInstanceOf[LArray[A]] - case _ => emptyObjectArray.asInstanceOf[LArray[A]] + case jl.Short.TYPE => emptyShortArray.asInstanceOf[LArray[A]] + case jl.Long.TYPE => emptyLongArray.asInstanceOf[LArray[A]] + case jl.Float.TYPE => emptyFloatArray.asInstanceOf[LArray[A]] + case jl.Double.TYPE => emptyDoubleArray.asInstanceOf[LArray[A]] + case jl.Boolean.TYPE => emptyBooleanArray.asInstanceOf[LArray[A]] + case _ => emptyObjectArray.asInstanceOf[LArray[A]] } } - /** - * Wrap a raw memory with a new LArray class - * @param byteSize the byte size of memory - * @param m the raw memory address - * @tparam A the element type of the array - * @return new instance of LArray[A] - */ + * Wrap a raw memory with a new LArray class + * @param byteSize + * the byte size of memory + * @param m + * the raw memory address + * @tparam A + * the element type of the array + * @return + * new instance of LArray[A] + */ private[larray] def wrap[A: ClassTag](byteSize: Long, m: Memory): LArray[A] = { val tag = implicitly[ClassTag[A]] tag.runtimeClass match { - case jl.Integer.TYPE => new LIntArray(byteSize / 4, m).asInstanceOf[LArray[A]] - case jl.Byte.TYPE => new LByteArray(byteSize, m).asInstanceOf[LArray[A]] - case jl.Long.TYPE => new LLongArray(byteSize / 8, m).asInstanceOf[LArray[A]] + case jl.Integer.TYPE => new LIntArray(byteSize / 4, m).asInstanceOf[LArray[A]] + case jl.Byte.TYPE => new LByteArray(byteSize, m).asInstanceOf[LArray[A]] + case jl.Long.TYPE => new LLongArray(byteSize / 8, m).asInstanceOf[LArray[A]] case jl.Character.TYPE => new LCharArray(byteSize / 2, m).asInstanceOf[LArray[A]] - case jl.Short.TYPE => new LShortArray(byteSize / 2, m).asInstanceOf[LArray[A]] - case jl.Float.TYPE => new LFloatArray(byteSize / 4, m).asInstanceOf[LArray[A]] - case jl.Double.TYPE => new LDoubleArray(byteSize / 8, m).asInstanceOf[LArray[A]] - //case jl.Boolean.TYPE => new LBitArray(byteSize * 8, m).asInstanceOf[LArray[A]] + case jl.Short.TYPE => new LShortArray(byteSize / 2, m).asInstanceOf[LArray[A]] + case jl.Float.TYPE => new LFloatArray(byteSize / 4, m).asInstanceOf[LArray[A]] + case jl.Double.TYPE => new LDoubleArray(byteSize / 8, m).asInstanceOf[LArray[A]] + // case jl.Boolean.TYPE => new LBitArray(byteSize * 8, m).asInstanceOf[LArray[A]] case _ => sys.error(s"unsupported type: $tag") } } - /** - * Convert iterator to LArray - * @param it - * @tparam A - * @return - */ + * Convert iterator to LArray + * @param it + * @tparam A + * @return + */ def toLArray[A: ClassTag](it: LIterator[A]): LArray[A] = { val b = newBuilder[A] it.foreach(b += _) @@ -465,15 +473,17 @@ object LArray { } /** - * Creates an LArray with given elements. - * - * @param xs the elements to put in the array - * @return an array containing all elements from xs. - */ + * Creates an LArray with given elements. + * + * @param xs + * the elements to put in the array + * @return + * an array containing all elements from xs. + */ def apply[A: ClassTag](xs: A*): LArray[A] = { val size = xs.size - val arr = new LObjectArray32[A](size) - var i = 0 + val arr = new LObjectArray32[A](size) + var i = 0 for (x <- xs) { arr(i) = x; i += 1 } @@ -482,7 +492,7 @@ object LArray { def apply(first: Byte, elems: Byte*): LArray[Byte] = { val size = 1 + elems.size - val arr = new LByteArray(size) + val arr = new LByteArray(size) arr(0) = first for ((e, i) <- elems.zipWithIndex) { arr(i + 1) = e @@ -490,11 +500,10 @@ object LArray { arr } - def apply(first: Int, elems: Int*): LArray[Int] = { // elems: Int* => Seq[Int] val size = 1 + elems.size - val arr = new LIntArray(size) + val arr = new LIntArray(size) // Populate the array elements arr(0) = first for ((e, i) <- elems.zipWithIndex) { @@ -503,10 +512,9 @@ object LArray { arr } - def apply(first: Char, elems: Char*): LArray[Char] = { val size = 1 + elems.size - val arr = new LCharArray(size) + val arr = new LCharArray(size) arr(0) = first for ((e, i) <- elems.zipWithIndex) { arr(i + 1) = e @@ -516,7 +524,7 @@ object LArray { def apply(first: Short, elems: Short*): LArray[Short] = { val size = 1 + elems.size - val arr = new LShortArray(size) + val arr = new LShortArray(size) arr(0) = first for ((e, i) <- elems.zipWithIndex) { arr(i + 1) = e @@ -526,7 +534,7 @@ object LArray { def apply(first: Float, elems: Float*): LArray[Float] = { val size = 1 + elems.size - val arr = new LFloatArray(size) + val arr = new LFloatArray(size) arr(0) = first for ((e, i) <- elems.zipWithIndex) { arr(i + 1) = e @@ -534,10 +542,9 @@ object LArray { arr } - def apply(first: Double, elems: Double*): LArray[Double] = { val size = 1 + elems.size - val arr = new LDoubleArray(size) + val arr = new LDoubleArray(size) arr(0) = first for ((e, i) <- elems.zipWithIndex) { arr(i + 1) = e @@ -547,7 +554,7 @@ object LArray { def apply(first: Long, elems: Long*): LArray[Long] = { val size = 1 + elems.size - val arr = new LLongArray(size) + val arr = new LLongArray(size) arr(0) = first for ((e, i) <- elems.zipWithIndex) { arr(i + 1) = e @@ -555,17 +562,23 @@ object LArray { arr } - def copy[A](src:LArray[A], dest:LArray[A]) { copy(src, 0L, dest, 0L, src.size) } + def copy[A](src: LArray[A], dest: LArray[A]) { copy(src, 0L, dest, 0L, src.size) } /** - * Copy LArray elements to the destination LArray - * @param src the source LArray - * @param srcPos the element index to start copy - * @param dest the destination LArray - * @param destPos the element index of the destination array to start copy - * @param length the number of elements to copy - * @tparam A the element type - */ + * Copy LArray elements to the destination LArray + * @param src + * the source LArray + * @param srcPos + * the element index to start copy + * @param dest + * the destination LArray + * @param destPos + * the element index of the destination array to start copy + * @param length + * the number of elements to copy + * @tparam A + * the element type + */ def copy[A](src: LArray[A], srcPos: Long, dest: LArray[A], destPos: Long, length: Long) { import UnsafeUtil.unsafe val copyLen = math.min(length, math.min(src.size - srcPos, dest.size - destPos)) @@ -586,25 +599,22 @@ object LArray { } /** - * Create a new LArrayBuilder[A] - * @tparam A - * @return - */ + * Create a new LArrayBuilder[A] + * @tparam A + * @return + */ def newBuilder[A: ClassTag]: LBuilder[A, LArray[A]] = LArrayBuilder.make[A] - /** Creates LArary with given dimensions */ def ofDim[A: ClassTag](size: Long) = LArray.of[A](size) - /** - * Creates a 2-dimensional array. Returned LArray[LArray[A]] cannot be released immediately. If you need - * relesable arrays, use [[xerial.larray.LArray2D]]. - * - **/ + * Creates a 2-dimensional array. Returned LArray[LArray[A]] cannot be released immediately. If you need relesable + * arrays, use [[xerial.larray.LArray2D]]. + */ def ofDim[A: ClassTag](n1: Long, n2: Long): LArray[LArray[A]] = { val arr: LArray[LArray[A]] = LArray.of[LArray[A]](n1) - var i = 0L + var i = 0L while (i < n1) { arr(i) = LArray.of[A](n2) i += 1 @@ -624,7 +634,6 @@ object LArray { def ofDim[A: ClassTag](n1: Long, n2: Long, n3: Long, n4: Long, n5: Long): LArray[LArray[LArray[LArray[LArray[A]]]]] = tabulate(n1)(_ => ofDim[A](n2, n3, n4, n5)) - def tabulate[A: ClassTag](n: Long)(f: Long => A): LArray[A] = { val b = newBuilder[A] b.sizeHint(n) @@ -636,43 +645,45 @@ object LArray { b.result } - - /** - * Create a LArray[Byte] of a memory mapped file - * @param f file - * @param offset offset in file - * @param size region byte size - * @param mode open mode. - */ - def mmap(f:File, offset:Long, size:Long, mode:MMapMode) : MappedLByteArray = { - new MappedLByteArray(f, offset, size, mode) - } - - /** - * Create a LArray[Byte] of a memory mapped file - * @param f file - * @param mode open mode. - */ - def mmap(f:File, mode:MMapMode) : MappedLByteArray = { - new MappedLByteArray(f, 0, f.length(), mode) - } +// /** +// * Create a LArray[Byte] of a memory mapped file +// * @param f +// * file +// * @param offset +// * offset in file +// * @param size +// * region byte size +// * @param mode +// * open mode. +// */ +// def mmap(f: File, offset: Long, size: Long, mode: MMapMode): MappedLByteArray = { +// new MappedLByteArray(f, offset, size, mode) +// } +// +// /** +// * Create a LArray[Byte] of a memory mapped file +// * @param f +// * file +// * @param mode +// * open mode. +// */ +// def mmap(f: File, mode: MMapMode): MappedLByteArray = { +// new MappedLByteArray(f, 0, f.length(), mode) +// } } /** - * read/write operations that can be supported for LArrays using raw byte arrays as their back-end. - */ + * read/write operations that can be supported for LArrays using raw byte arrays as their back-end. + */ trait RawByteArray[A] extends LArray[A] { - - /** - * Create an input stream for reading LArray byte contents - * @return - */ + * Create an input stream for reading LArray byte contents + * @return + */ def toInputStream: java.io.InputStream = LArrayInputStream(this) - import UnsafeUtil.unsafe def clear() { @@ -682,7 +693,7 @@ trait RawByteArray[A] extends LArray[A] { protected var cursor = 0L override def write(src: ByteBuffer): Int = { - val len = math.max(src.limit - src.position, 0) + val len = math.max(src.limit() - src.position(), 0) val writeLen = src match { case d: DirectBuffer => unsafe.copyMemory(d.address() + src.position(), address + cursor, len) @@ -698,36 +709,44 @@ trait RawByteArray[A] extends LArray[A] { len } cursor += writeLen - src.position(src.position + writeLen) + src.position(src.position() + writeLen) writeLen } - /** - * Write the contents of this array to the destination buffer - * @param srcOffset byte offset - * @param dest destination array - * @param destOffset offset in the destination array - * @param length the byte length to write - * @return written byte length - */ + * Write the contents of this array to the destination buffer + * @param srcOffset + * byte offset + * @param dest + * destination array + * @param destOffset + * offset in the destination array + * @param length + * the byte length to write + * @return + * written byte length + */ def writeToArray(srcOffset: Long, dest: Array[Byte], destOffset: Int, length: Int): Int = { val writeLen = math.min(dest.length - destOffset, math.min(length, byteLength - srcOffset)).toInt - val b = xerial.larray.buffer.UnsafeUtil.newDirectByteBuffer(address + srcOffset, writeLen, this) + val b = xerial.larray.buffer.UnsafeUtil.newDirectByteBuffer(address + srcOffset, writeLen, this) b.get(dest, destOffset, writeLen) writeLen } /** - * Read the contents from a given source buffer - * @param src source buffer - * @param srcOffset byte offset in the source buffer - * @param destOffset byte offset from the destination address - * @param length byte length to read from the source - */ + * Read the contents from a given source buffer + * @param src + * source buffer + * @param srcOffset + * byte offset in the source buffer + * @param destOffset + * byte offset from the destination address + * @param length + * byte length to read from the source + */ def readFromArray(src: Array[Byte], srcOffset: Int, destOffset: Long, length: Int): Int = { val readLen = math.min(src.length - srcOffset, math.min(byteLength - destOffset, length)).toInt - val b = xerial.larray.buffer.UnsafeUtil.newDirectByteBuffer(address + destOffset, readLen, this) + val b = xerial.larray.buffer.UnsafeUtil.newDirectByteBuffer(address + destOffset, readLen, this) b.put(src, srcOffset, readLen) readLen } @@ -740,29 +759,27 @@ trait RawByteArray[A] extends LArray[A] { unsafe.copyMemory(address + srcOffset, dst.address + dstOffset, blen) } - } - private[larray] trait UnsafeArray[T] extends RawByteArray[T] with LogSupport { self: LArray[T] => private[larray] def m: Memory def address = m.address - private[larray] def alloc :MemoryAllocator + private[larray] def alloc: MemoryAllocator /** - * Release the memory of LArray. After calling this method, the results of calling the behavior of the other methods becomes undefined or might cause JVM crash. - */ + * Release the memory of LArray. After calling this method, the results of calling the behavior of the other methods + * becomes undefined or might cause JVM crash. + */ def free { alloc.release(m) } } - class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Char] - with UnsafeArray[Char] { + extends LArray[Char] + with UnsafeArray[Char] { protected[this] def newBuilder = new LCharArrayBuilder def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 1))(alloc) @@ -780,8 +797,8 @@ class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val all } /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize: Int = 2 def view(from: Long, to: Long) = new LArrayView.LCharArrayView(this, from, to - from) @@ -789,14 +806,17 @@ class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val all } /** - * LArray of Int type - * @param size the size of array - * @param m allocated memory - * @param alloc memory allocator - */ + * LArray of Int type + * @param size + * the size of array + * @param m + * allocated memory + * @param alloc + * memory allocator + */ class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Int] - with UnsafeArray[Int] { + extends LArray[Int] + with UnsafeArray[Int] { protected[this] def newBuilder = new LIntArrayBuilder def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) @@ -814,8 +834,8 @@ class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val allo } /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize: Int = 4 def view(from: Long, to: Long) = new LArrayView.LIntArrayView(this, from, to - from) @@ -823,14 +843,17 @@ class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val allo } /** - * LArray of Long type - * @param size the size of array - * @param m allocated memory - * @param alloc memory allocator - */ + * LArray of Long type + * @param size + * the size of array + * @param m + * allocated memory + * @param alloc + * memory allocator + */ class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Long] - with UnsafeArray[Long] { + extends LArray[Long] + with UnsafeArray[Long] { def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 3))(alloc) protected[this] def newBuilder = new LLongArrayBuilder @@ -853,16 +876,18 @@ class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val all } - /** - * LArray of Byte type - * @param size the size of array - * @param m allocated memory - * @param alloc memory allocator - */ + * LArray of Byte type + * @param size + * the size of array + * @param m + * allocated memory + * @param alloc + * memory allocator + */ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Byte] - with UnsafeArray[Byte] { + extends LArray[Byte] + with UnsafeArray[Byte] { self => def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size))(alloc) @@ -872,35 +897,43 @@ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val all private[larray] def elementByteSize: Int = 1 /** - * Retrieve an element - * @param i index - * @return the element value - */ + * Retrieve an element + * @param i + * index + * @return + * the element value + */ def apply(i: Long): Byte = { UnsafeUtil.unsafe.getByte(m.address + i) } - def apply(i: Int) : Byte = { + def apply(i: Int): Byte = { UnsafeUtil.unsafe.getByte(m.address + i) } /** - * Update an element - * @param i index to be updated - * @param v value to set - * @return the value - */ + * Update an element + * @param i + * index to be updated + * @param v + * value to set + * @return + * the value + */ def update(i: Long, v: Byte): Byte = { UnsafeUtil.unsafe.putByte(m.address + i, v) v } /** - * Update an element - * @param i index to be updated - * @param v value to set - * @return the value - */ + * Update an element + * @param i + * index to be updated + * @param v + * value to set + * @return + * the value + */ def update(i: Int, v: Byte): Byte = { UnsafeUtil.unsafe.putByte(m.address + i, v) v @@ -915,7 +948,7 @@ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val all { var i: Long = left - 1 - while ( { + while ({ i += 1; i <= right }) { count(self(i) - Byte.MinValue) += 1 @@ -923,14 +956,14 @@ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val all } { - var i = NUM_BYTE_VALUES + var i = NUM_BYTE_VALUES var k: Long = right + 1 while (k > left) { - while ( { + while ({ i -= 1; count(i) == 0 }) {} val value: Byte = (i + Byte.MinValue).toByte - var s = count(i) + var s = count(i) do { k -= 1 self(k) = value @@ -944,17 +977,15 @@ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val all sort(0L, size - 1L) } - def view(from: Long, to: Long) = new LArrayView.LByteArrayView(this, from, to - from) } class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Double] - with UnsafeArray[Double] { + extends LArray[Double] + with UnsafeArray[Double] { def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 3))(alloc) - private[larray] def elementByteSize = 8 import UnsafeUtil.unsafe @@ -975,8 +1006,8 @@ class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val a } class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Float] - with UnsafeArray[Float] { + extends LArray[Float] + with UnsafeArray[Float] { def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) private[larray] def elementByteSize = 4 @@ -999,8 +1030,8 @@ class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val al } class LShortArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) - extends LArray[Short] - with UnsafeArray[Short] { + extends LArray[Short] + with UnsafeArray[Short] { def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 1))(alloc) private[larray] def elementByteSize = 2 @@ -1022,7 +1053,6 @@ class LShortArray(val size: Long, private[larray] val m: Memory)(implicit val al def view(from: Long, to: Long) = new LArrayView.LShortArrayView(this, from, to - from) } - object LObjectArray { def ofDim[A: ClassTag](size: Long) = if (size < Int.MaxValue) @@ -1032,17 +1062,19 @@ object LObjectArray { } /** - * LArray[A] of Objects. This implementation is a simple wrapper of Array[A] and used when the array size is less than 2G - * @param size array size - * @tparam A object type - */ + * LArray[A] of Objects. This implementation is a simple wrapper of Array[A] and used when the array size is less than + * 2G + * @param size + * array size + * @tparam A + * object type + */ class LObjectArray32[A: ClassTag](val size: Long) extends LArray[A] { require(size < Int.MaxValue) private var array = new Array[A](size.toInt) protected[this] def newBuilder = new LObjectArrayBuilder[A] - def clear() { java.util.Arrays.fill(array.asInstanceOf[Array[AnyRef]], 0, length.toInt, null) } @@ -1066,12 +1098,13 @@ class LObjectArray32[A: ClassTag](val size: Long) extends LArray[A] { } /** - * Copy the contents of this sequence into the target LByteArray - * @param srcOffset - * @param dst - * @param dstOffset - * @param blen the byte length to copy - */ + * Copy the contents of this sequence into the target LByteArray + * @param srcOffset + * @param dst + * @param dstOffset + * @param blen + * the byte length to copy + */ def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { throw new UnsupportedOperationException("copyTo(Long, LByteArray, Long, Long)") } @@ -1079,24 +1112,26 @@ class LObjectArray32[A: ClassTag](val size: Long) extends LArray[A] { def view(from: Long, to: Long) = new LArrayView.LObjectArrayView[A](this, from, to - from) /** - * Raw-memory address of this array - */ + * Raw-memory address of this array + */ def address = LArray.EmptyArray.address } /** - * LArray[A] of Object of more than 2G entries. - * @param size array size - * @tparam A object type - */ + * LArray[A] of Object of more than 2G entries. + * @param size + * array size + * @tparam A + * object type + */ class LObjectArrayLarge[A: ClassTag](val size: Long) extends LArray[A] { protected[this] def newBuilder = new LObjectArrayBuilder[A] /** - * block size in pow(2, B) - */ - private val B = 31 + * block size in pow(2, B) + */ + private val B = 31 private val mask = (1L << B) - 1L @inline private def index(i: Long): Int = (i >>> B).toInt @@ -1107,7 +1142,7 @@ class LObjectArrayLarge[A: ClassTag](val size: Long) extends LArray[A] { val BLOCK_SIZE = (1L << B).toInt val NUM_BLOCKS = index(size - 1) + 1 // initialize the array - val a = new Array[Array[A]](NUM_BLOCKS) + val a = new Array[Array[A]](NUM_BLOCKS) var remaining = size for (i <- 0 until NUM_BLOCKS) { val s = math.min(remaining, BLOCK_SIZE).toInt @@ -1132,7 +1167,6 @@ class LObjectArrayLarge[A: ClassTag](val size: Long) extends LArray[A] { throw new UnsupportedOperationException("copyTo(Long, LByteArray, Long, Long)") } - def apply(i: Long) = array(index(i))(offset(i)) def update(i: Long, v: A) = { @@ -1148,4 +1182,4 @@ class LObjectArrayLarge[A: ClassTag](val size: Long) extends LArray[A] { def view(from: Long, to: Long) = new LArrayView.LObjectArrayView[A](this, from, to - from) def address = LArray.EmptyArray.address -} \ No newline at end of file +} diff --git a/larray/src/main/scala/xerial/larray/LArray2D.scala b/larray/src/main/scala/xerial/larray/LArray2D.scala index 05e76c2..df9e01a 100644 --- a/larray/src/main/scala/xerial/larray/LArray2D.scala +++ b/larray/src/main/scala/xerial/larray/LArray2D.scala @@ -27,49 +27,56 @@ import reflect.ClassTag object LArray2D { /** - * Create a new 2D array - * @param rowSize the row size - * @param colSize the column size - * @tparam A the element type - * @return a new 2D array - */ - def of[A : ClassTag](rowSize:Long, colSize:Long) = new LArray2D[A](rowSize, colSize) + * Create a new 2D array + * @param rowSize + * the row size + * @param colSize + * the column size + * @tparam A + * the element type + * @return + * a new 2D array + */ + def of[A: ClassTag](rowSize: Long, colSize: Long) = new LArray2D[A](rowSize, colSize) } /** - * LArray2D is a wrapper of LArray to emulate 2-dimensional array using a single array. - * @author Taro L. Saito - */ -class LArray2D[A : ClassTag](val rowSize:Long, val colSize:Long) { - private val arr : LArray[A] = LArray.of[A](rowSize * colSize) + * LArray2D is a wrapper of LArray to emulate 2-dimensional array using a single array. + * @author + * Taro L. Saito + */ +class LArray2D[A: ClassTag](val rowSize: Long, val colSize: Long) { + private val arr: LArray[A] = LArray.of[A](rowSize * colSize) - @inline def pos(i:Long, j:Long) = i * rowSize + j + @inline def pos(i: Long, j: Long) = i * rowSize + j - def apply(i:Long, j:Long) : A = arr(pos(i, j)) - def update(i:Long, j:Long, v:A) : A = arr.update(pos(i,j), v) + def apply(i: Long, j: Long): A = arr(pos(i, j)) + def update(i: Long, j: Long, v: A): A = arr.update(pos(i, j), v) def free { arr.free } /** - * Get a view of a row - * @param row the row index - * @return LArrayView[A] of the specified row - */ - def row(row:Long) = { + * Get a view of a row + * @param row + * the row index + * @return + * LArrayView[A] of the specified row + */ + def row(row: Long) = { val p = pos(row, 0) - arr.view(p, p+colSize) + arr.view(p, p + colSize) } /** - * Clear all of the elements in the array - */ + * Clear all of the elements in the array + */ def clear() = arr.clear() /** - * Get a raw LArray representation of this 2D array - * @return - */ - def rawArray : LArray[A] = arr + * Get a raw LArray representation of this 2D array + * @return + */ + def rawArray: LArray[A] = arr -} \ No newline at end of file +} diff --git a/larray/src/main/scala/xerial/larray/LArrayBuilder.scala b/larray/src/main/scala/xerial/larray/LArrayBuilder.scala index 86403ab..21d0f87 100644 --- a/larray/src/main/scala/xerial/larray/LArrayBuilder.scala +++ b/larray/src/main/scala/xerial/larray/LArrayBuilder.scala @@ -34,8 +34,10 @@ import scala.reflect.ClassTag /** * Extension of `scala.collection.mutable.Builder` using Long indexes * - * @tparam Elem element type - * @tparam To LArray type to generate + * @tparam Elem + * element type + * @tparam To + * LArray type to generate */ trait LBuilder[Elem, +To] extends WritableByteChannel { @@ -45,58 +47,65 @@ trait LBuilder[Elem, +To] extends WritableByteChannel { def append(seq: LSeq[Elem]): this.type - /** Adds a single element to the builder. + /** + * Adds a single element to the builder. * - * @param elem the element to be added. - * @return the builder itself. + * @param elem + * the element to be added. + * @return + * the builder itself. */ def +=(elem: Elem): this.type - /** Adds all elements produced by a TraversableOnce to this coll. + /** + * Adds all elements produced by a TraversableOnce to this coll. * - * @param xs the TraversableOnce producing the elements to add. - * @return the coll itself. + * @param xs + * the TraversableOnce producing the elements to add. + * @return + * the coll itself. */ - def ++=(xs: TraversableOnce[Elem]): this.type = {xs.seq foreach +=; this} + def ++=(xs: IterableOnce[Elem]): this.type = { xs foreach +=; this } - def ++=(xs: LIterator[Elem]): this.type = {xs foreach +=; this} + def ++=(xs: LIterator[Elem]): this.type = { xs foreach +=; this } - /** Clears the contents of this builder. - * After execution of this method the builder will contain no elements. + /** + * Clears the contents of this builder. After execution of this method the builder will contain no elements. */ def clear() - /** Produces a collection from the added elements. - * The builder's contents are undefined after this operation. + /** + * Produces a collection from the added elements. The builder's contents are undefined after this operation. * - * @return a collection containing the elements added to this builder. + * @return + * a collection containing the elements added to this builder. */ def result(): To - /** Gives a hint how many elements are expected to be added - * when the next `result` is called. Some builder classes - * will optimize their representation based on the hint. However, - * builder implementations are still required to work correctly even if the hint is - * wrong, i.e. a different number of elements is added. + /** + * Gives a hint how many elements are expected to be added when the next `result` is called. Some builder classes + * will optimize their representation based on the hint. However, builder implementations are still required to work + * correctly even if the hint is wrong, i.e. a different number of elements is added. * - * @param size the hint how many elements will be added. + * @param size + * the hint how many elements will be added. */ def sizeHint(size: Long): Unit } /** - * In the following, we need to define builders for every primitive types because if we extract - * common functions (e.g., resize, mkArray) using type parameter, we cannot avoid boxing/unboxing. - * + * In the following, we need to define builders for every primitive types because if we extract common functions (e.g., + * resize, mkArray) using type parameter, we cannot avoid boxing/unboxing. */ abstract class LArrayBuilder[A, Repr <: LArray[A]] extends LBuilder[A, Repr] with LogSupport { - protected var elems : LByteArray = _ - protected var capacity: Long = 0L + protected var elems: LByteArray = _ + protected var capacity: Long = 0L + /** * Current cursor position in LByteArray */ - protected var cursor : Long = 0L + protected var cursor: Long = 0L protected def numElems: Long = cursor / elementSize @@ -134,8 +143,7 @@ abstract class LArrayBuilder[A, Repr <: LArray[A]] extends LBuilder[A, Repr] wit if (capacity < size || capacity == 0L) { var newsize = if (capacity <= 1L) { 16L - } - else { + } else { capacity * factor } while (newsize < size) newsize *= factor @@ -158,12 +166,12 @@ abstract class LArrayBuilder[A, Repr <: LArray[A]] extends LBuilder[A, Repr] wit def write(src: ByteBuffer): Int = { import UnsafeUtil.unsafe - val len = math.max(src.limit - src.position, 0) + val len = math.max(src.limit() - src.position(), 0) val toAdd = (len + elementSize - 1) / elementSize ensureSize(numElems + toAdd) val writeLen = src match { case d: DirectBuffer => - unsafe.copyMemory(d.address() + d.position, elems.address + cursor, len) + unsafe.copyMemory(d.address() + d.position(), elems.address + cursor, len) len case arr if src.hasArray => elems.readFromArray(src.array(), src.position(), cursor, len) @@ -177,13 +185,13 @@ abstract class LArrayBuilder[A, Repr <: LArray[A]] extends LBuilder[A, Repr] wit len } cursor += writeLen - src.position(src.position + writeLen) + src.position(src.position() + writeLen) writeLen } def isOpen: Boolean = true - def close() {clear()} + def close() { clear() } } class LByteArrayBuilder extends LArrayBuilder[Byte, LByteArray] { @@ -200,8 +208,7 @@ class LByteArrayBuilder extends LArrayBuilder[Byte, LByteArray] { def result(): LByteArray = { if (capacity != 0L && capacity == numElems) { elems - } - else { + } else { mkArray(numElems) } } @@ -219,8 +226,7 @@ class LCharArrayBuilder extends LArrayBuilder[Char, LCharArray] { def result(): LCharArray = { if (capacity != 0L && capacity == numElems) { new LCharArray(numElems, elems.m) - } - else { + } else { new LCharArray(numElems, mkArray(numElems).m) } } @@ -238,8 +244,7 @@ class LShortArrayBuilder extends LArrayBuilder[Short, LShortArray] { def result(): LShortArray = { if (capacity != 0L && capacity == numElems) { new LShortArray(numElems, elems.m) - } - else { + } else { new LShortArray(numElems, mkArray(numElems).m) } } @@ -258,8 +263,7 @@ class LIntArrayBuilder extends LArrayBuilder[Int, LIntArray] { def result(): LIntArray = { if (capacity != 0L && capacity == numElems) { new LIntArray(numElems, elems.m) - } - else { + } else { new LIntArray(numElems, mkArray(numElems).m) } } @@ -278,8 +282,7 @@ class LFloatArrayBuilder extends LArrayBuilder[Float, LFloatArray] { def result(): LFloatArray = { if (capacity != 0L && capacity == numElems) { new LFloatArray(numElems, elems.m) - } - else { + } else { new LFloatArray(numElems, mkArray(numElems).m) } } @@ -298,8 +301,7 @@ class LLongArrayBuilder extends LArrayBuilder[Long, LLongArray] { def result(): LLongArray = { if (capacity != 0L && capacity == numElems) { new LLongArray(numElems, elems.m) - } - else { + } else { new LLongArray(numElems, mkArray(numElems).m) } } @@ -319,8 +321,7 @@ class LDoubleArrayBuilder extends LArrayBuilder[Double, LDoubleArray] { def result(): LDoubleArray = { if (capacity != 0L && capacity == numElems) { new LDoubleArray(numElems, elems.m) - } - else { + } else { new LDoubleArray(numElems, mkArray(numElems).m) } } @@ -330,9 +331,9 @@ class LObjectArrayBuilder[A: ClassTag] extends LBuilder[A, LArray[A]] { def elementSize = 4 - private var elems : LArray[A] = _ - private var capacity: Long = 0L - private[larray] var size : Long = 0L + private var elems: LArray[A] = _ + private var capacity: Long = 0L + private[larray] var size: Long = 0L private def mkArray(size: Long): LArray[A] = { val newArray = LObjectArray.ofDim[A](size) @@ -352,8 +353,7 @@ class LObjectArrayBuilder[A: ClassTag] extends LBuilder[A, LArray[A]] { if (capacity < size || capacity == 0L) { var newsize = if (capacity <= 1L) { 16L - } - else { + } else { capacity * factor } while (newsize < size) newsize *= factor @@ -382,8 +382,7 @@ class LObjectArrayBuilder[A: ClassTag] extends LBuilder[A, LArray[A]] { def result(): LArray[A] = { if (capacity != 0L && capacity == size) { elems - } - else { + } else { mkArray(size) } } @@ -392,38 +391,42 @@ class LObjectArrayBuilder[A: ClassTag] extends LBuilder[A, LArray[A]] { def isOpen: Boolean = true - def close() {clear} + def close() { clear } def append(seq: LSeq[A]) = { ensureSize(size + seq.length) - seq.foreach {e => elems(size) = e; size += 1} + seq.foreach { e => elems(size) = e; size += 1 } this } } /** - * @author Taro L. Saito + * @author + * Taro L. Saito */ object LArrayBuilder { - /** Creates a new arraybuilder of type `T`. + /** + * Creates a new arraybuilder of type `T`. * - * @tparam T type of the elements for the array builder, with a `ClassTag` context bound. - * @return a new empty array builder. + * @tparam T + * type of the elements for the array builder, with a `ClassTag` context bound. + * @return + * a new empty array builder. */ def make[T: ClassTag](): LBuilder[T, LArray[T]] = { val tag = implicitly[ClassTag[T]] tag.runtimeClass match { - case java.lang.Byte.TYPE => new LByteArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case java.lang.Short.TYPE => new LShortArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Byte.TYPE => new LByteArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Short.TYPE => new LShortArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] case java.lang.Character.TYPE => new LCharArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case java.lang.Integer.TYPE => new LIntArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case java.lang.Long.TYPE => new LLongArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case java.lang.Float.TYPE => new LFloatArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case java.lang.Double.TYPE => new LDoubleArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case java.lang.Boolean.TYPE => new LBitArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] - case _ => new LObjectArrayBuilder[T].asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Integer.TYPE => new LIntArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Long.TYPE => new LLongArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Float.TYPE => new LFloatArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Double.TYPE => new LDoubleArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case java.lang.Boolean.TYPE => new LBitArrayBuilder().asInstanceOf[LBuilder[T, LArray[T]]] + case _ => new LObjectArrayBuilder[T].asInstanceOf[LBuilder[T, LArray[T]]] } } -} \ No newline at end of file +} diff --git a/larray/src/main/scala/xerial/larray/LArrayInputStream.scala b/larray/src/main/scala/xerial/larray/LArrayInputStream.scala index cfeaf6e..ab71de0 100644 --- a/larray/src/main/scala/xerial/larray/LArrayInputStream.scala +++ b/larray/src/main/scala/xerial/larray/LArrayInputStream.scala @@ -31,14 +31,17 @@ object LArrayInputStream { /** * Create a new InputStream from a given LArray * - * @param array input - * @tparam A element type - * @return input stream + * @param array + * input + * @tparam A + * element type + * @return + * input stream */ def apply[A](array: LArray[A]): InputStream = { array match { case r: RawByteArray[A] => new RawLArrayInputStream[A](r) - case _ => sys.error(s"cannot create InputStream from this LArray class:${array.getClass}") + case _ => sys.error(s"cannot create InputStream from this LArray class:${array.getClass}") } } @@ -47,7 +50,8 @@ object LArrayInputStream { /** * InputStream implementation for LArrays that uses RawByteArray internally. * - * @author Taro L. Saito + * @author + * Taro L. Saito */ private[larray] class RawLArrayInputStream[A](array: RawByteArray[A]) extends InputStream with LogSupport { @@ -63,8 +67,7 @@ private[larray] class RawLArrayInputStream[A](array: RawByteArray[A]) extends In override def read(b: Array[Byte], offset: Int, len: Int): Int = { if (cursor >= array.size) { -1 - } - else { + } else { val readLen = math.min(len, array.byteLength - cursor).toInt array.writeToArray(cursor, b, offset, readLen) cursor += readLen @@ -89,4 +92,4 @@ private[larray] class RawLArrayInputStream[A](array: RawByteArray[A]) extends In override def markSupported() = { true } -} \ No newline at end of file +} diff --git a/larray/src/main/scala/xerial/larray/LArrayOutputStream.scala b/larray/src/main/scala/xerial/larray/LArrayOutputStream.scala index 55bf940..bf2bc73 100644 --- a/larray/src/main/scala/xerial/larray/LArrayOutputStream.scala +++ b/larray/src/main/scala/xerial/larray/LArrayOutputStream.scala @@ -26,11 +26,12 @@ import java.io.OutputStream import reflect.ClassTag /** - * Create LArray using `java.io.OutputStream` interface - * - * @author Taro L. Saito - */ -class LArrayOutputStream[A : ClassTag] extends OutputStream { + * Create LArray using `java.io.OutputStream` interface + * + * @author + * Taro L. Saito + */ +class LArrayOutputStream[A: ClassTag] extends OutputStream { private val buf = new LByteArrayBuilder @@ -42,9 +43,9 @@ class LArrayOutputStream[A : ClassTag] extends OutputStream { buf.append(b, off, len) } - def result : LArray[A] = { + def result: LArray[A] = { val arr = buf.result.asInstanceOf[LByteArray] LArray.wrap[A](arr.size, arr.m) } -} \ No newline at end of file +} diff --git a/larray/src/main/scala/xerial/larray/LArrayView.scala b/larray/src/main/scala/xerial/larray/LArrayView.scala index 224df93..6f266e2 100644 --- a/larray/src/main/scala/xerial/larray/LArrayView.scala +++ b/larray/src/main/scala/xerial/larray/LArrayView.scala @@ -26,165 +26,177 @@ import reflect.ClassTag import xerial.larray.LArray.EmptyArray /** - * Provides view of LArray - * @author Taro L. Saito - */ + * Provides view of LArray + * @author + * Taro L. Saito + */ object LArrayView { - class LBitArrayView(base:LBitArray, offset:Long, val size:Long) extends LArrayView[Boolean] with LBitArrayOps { + class LBitArrayView(base: LBitArray, offset: Long, val size: Long) extends LArrayView[Boolean] with LBitArrayOps { protected[this] def newBuilder = new LBitArrayBuilder - def apply(i:Long) = base.apply(i + offset) + def apply(i: Long) = base.apply(i + offset) def address = LArray.EmptyArray.address /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize = throw new UnsupportedOperationException("elementByteSize of LBitArray") /** - * Copy the contents of this LSeq[A] into the target LByteArray - * @param dst - * @param dstOffset - */ + * Copy the contents of this LSeq[A] into the target LByteArray + * @param dst + * @param dstOffset + */ def copyTo(dst: LByteArray, dstOffset: Long) { throw new UnsupportedOperationException("copyTo of LBitArray") } /** - * Copy the contents of this sequence into the target LByteArray - * @param srcOffset - * @param dst - * @param dstOffset - * @param blen the byte length to copy - */ + * Copy the contents of this sequence into the target LByteArray + * @param srcOffset + * @param dst + * @param dstOffset + * @param blen + * the byte length to copy + */ def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { throw new UnsupportedOperationException("copyTo of LBitArray") } /** - * Count the number of bits within the specified range [start, end) - * @param checkTrue count true or false - * @param start - * @param end - * @return the number of occurrences - */ - def count(checkTrue: Boolean, start: Long, end: Long) = base.count(checkTrue, start+offset, end+offset) - - override def slice(from: Long, until: Long) = base.slice(from+offset, until+offset) + * Count the number of bits within the specified range [start, end) + * @param checkTrue + * count true or false + * @param start + * @param end + * @return + * the number of occurrences + */ + def count(checkTrue: Boolean, start: Long, end: Long) = base.count(checkTrue, start + offset, end + offset) + + override def slice(from: Long, until: Long) = base.slice(from + offset, until + offset) } - class LByteArrayView(base:LArray[Byte], offset:Long, size:Long) extends AbstractLArrayView[Byte](base, offset, size) { + class LByteArrayView(base: LArray[Byte], offset: Long, size: Long) + extends AbstractLArrayView[Byte](base, offset, size) { protected[this] def newBuilder: LBuilder[Byte, LArray[Byte]] = new LByteArrayBuilder - private[larray] def elementByteSize: Int = 1 + private[larray] def elementByteSize: Int = 1 } - class LCharArrayView(base:LArray[Char], offset:Long, size:Long) extends AbstractLArrayView[Char](base, offset, size) { + class LCharArrayView(base: LArray[Char], offset: Long, size: Long) + extends AbstractLArrayView[Char](base, offset, size) { protected[this] def newBuilder: LBuilder[Char, LArray[Char]] = new LCharArrayBuilder - private[larray] def elementByteSize: Int = 2 + private[larray] def elementByteSize: Int = 2 } - class LShortArrayView(base:LArray[Short], offset:Long, size:Long) extends AbstractLArrayView[Short](base, offset, size) { + class LShortArrayView(base: LArray[Short], offset: Long, size: Long) + extends AbstractLArrayView[Short](base, offset, size) { protected[this] def newBuilder: LBuilder[Short, LArray[Short]] = new LShortArrayBuilder - private[larray] def elementByteSize: Int = 2 + private[larray] def elementByteSize: Int = 2 } - class LIntArrayView(base:LArray[Int], offset:Long, size:Long) extends AbstractLArrayView[Int](base, offset, size) { + class LIntArrayView(base: LArray[Int], offset: Long, size: Long) extends AbstractLArrayView[Int](base, offset, size) { protected[this] def newBuilder: LBuilder[Int, LArray[Int]] = new LIntArrayBuilder - private[larray] def elementByteSize: Int = 4 + private[larray] def elementByteSize: Int = 4 } - class LFloatArrayView(base:LArray[Float], offset:Long, size:Long) extends AbstractLArrayView[Float](base, offset, size) { + class LFloatArrayView(base: LArray[Float], offset: Long, size: Long) + extends AbstractLArrayView[Float](base, offset, size) { protected[this] def newBuilder: LBuilder[Float, LArray[Float]] = new LFloatArrayBuilder - private[larray] def elementByteSize: Int = 4 + private[larray] def elementByteSize: Int = 4 } - class LLongArrayView(base:LArray[Long], offset:Long, size:Long) extends AbstractLArrayView[Long](base, offset, size) { + class LLongArrayView(base: LArray[Long], offset: Long, size: Long) + extends AbstractLArrayView[Long](base, offset, size) { protected[this] def newBuilder: LBuilder[Long, LArray[Long]] = new LLongArrayBuilder - private[larray] def elementByteSize: Int = 8 + private[larray] def elementByteSize: Int = 8 } - class LDoubleArrayView(base:LArray[Double], offset:Long, size:Long) extends AbstractLArrayView[Double](base, offset, size) { + class LDoubleArrayView(base: LArray[Double], offset: Long, size: Long) + extends AbstractLArrayView[Double](base, offset, size) { protected[this] def newBuilder: LBuilder[Double, LArray[Double]] = new LDoubleArrayBuilder - private[larray] def elementByteSize: Int = 8 + private[larray] def elementByteSize: Int = 8 } - class LObjectArrayView[A : ClassTag](base:LArray[A], offset:Long, size:Long) extends AbstractLArrayView[A](base, offset, size) { + class LObjectArrayView[A: ClassTag](base: LArray[A], offset: Long, size: Long) + extends AbstractLArrayView[A](base, offset, size) { protected[this] def newBuilder: LBuilder[A, LArray[A]] = new LObjectArrayBuilder - private[larray] def elementByteSize: Int = 4 + private[larray] def elementByteSize: Int = 4 } object EmptyView extends LArrayView[Nothing] { protected[this] def newBuilder = EmptyArray.newBuilder - def size = 0L - def apply(i: Long) = EmptyArray.apply(i) + def size = 0L + def apply(i: Long) = EmptyArray.apply(i) def address = LArray.EmptyArray.address /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize = EmptyArray.elementByteSize /** - * Copy the contents of this LSeq[A] into the target LByteArray - * @param dst - * @param dstOffset - */ - def copyTo(dst: LByteArray, dstOffset: Long) { EmptyArray.copyTo(dst, dstOffset)} + * Copy the contents of this LSeq[A] into the target LByteArray + * @param dst + * @param dstOffset + */ + def copyTo(dst: LByteArray, dstOffset: Long) { EmptyArray.copyTo(dst, dstOffset) } /** - * Copy the contents of this sequence into the target LByteArray - * @param srcOffset - * @param dst - * @param dstOffset - * @param blen the byte length to copy - */ - def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { EmptyArray.copyTo(srcOffset, dst, dstOffset, blen)} + * Copy the contents of this sequence into the target LByteArray + * @param srcOffset + * @param dst + * @param dstOffset + * @param blen + * the byte length to copy + */ + def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { + EmptyArray.copyTo(srcOffset, dst, dstOffset, blen) + } } } /** - * Shallow-copy reference of the part of LArray - * @tparam A - */ -trait LArrayView[A] extends LSeq[A] { - -} + * Shallow-copy reference of the part of LArray + * @tparam A + */ +trait LArrayView[A] extends LSeq[A] {} -abstract class AbstractLArrayView[A : ClassTag](base:LSeq[A], offset:Long, val size:Long) extends LArrayView[A] { +abstract class AbstractLArrayView[A: ClassTag](base: LSeq[A], offset: Long, val size: Long) extends LArrayView[A] { def address = base.address + (offset * elementByteSize) /** - * Retrieve an element - * @param i index - * @return the element value - */ + * Retrieve an element + * @param i + * index + * @return + * the element value + */ def apply(i: Long): A = base.apply(i + offset) /** - * Copy the contents of this LSeq[A] into the target LByteArray - * @param dst - * @param dstOffset - */ + * Copy the contents of this LSeq[A] into the target LByteArray + * @param dst + * @param dstOffset + */ def copyTo(dst: LByteArray, dstOffset: Long) { base.copyTo(offset, dst, dstOffset, byteLength) } /** - * Copy the contents of this sequence into the target LByteArray - * @param srcOffset - * @param dst - * @param dstOffset - * @param blen the byte length to copy - */ + * Copy the contents of this sequence into the target LByteArray + * @param srcOffset + * @param dst + * @param dstOffset + * @param blen + * the byte length to copy + */ def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { base.copyTo[B](offset + srcOffset, dst, dstOffset, blen) } - - } - diff --git a/larray/src/main/scala/xerial/larray/LBitArray.scala b/larray/src/main/scala/xerial/larray/LBitArray.scala index ec6f0f2..b6d5b38 100644 --- a/larray/src/main/scala/xerial/larray/LBitArray.scala +++ b/larray/src/main/scala/xerial/larray/LBitArray.scala @@ -31,19 +31,20 @@ import xerial.larray.buffer.Memory /** * Helper methods for packing bit sequences into Array[Long] * - * @author leo + * @author + * leo */ object BitEncoder { def minArraySize(numBits: Long): Long = { val blockBitSize: Long = 64L - val arraySize: Long = (numBits + blockBitSize - 1L) / blockBitSize + val arraySize: Long = (numBits + blockBitSize - 1L) / blockBitSize arraySize } - @inline def blockAddr(pos: Long): Long = blockIndex(pos) << 3 + @inline def blockAddr(pos: Long): Long = blockIndex(pos) << 3 @inline def blockIndex(pos: Long): Long = (pos >>> 6) - @inline def blockOffset(pos: Long): Int = (pos & 0x3FL).toInt + @inline def blockOffset(pos: Long): Int = (pos & 0x3fL).toInt val table = Array(false, true) } @@ -62,11 +63,10 @@ object LBitArray { b.sizeHint(bitString.length) for (ch <- bitString) { b += (if (ch == '0') { - true - } - else { - false - }) + true + } else { + false + }) } b.result() } @@ -78,10 +78,12 @@ trait LBitArrayOps { /** * Count the number of bits within the specified range [start, end) * - * @param checkTrue count true or false + * @param checkTrue + * count true or false * @param start * @param end - * @return the number of occurrences + * @return + * the number of occurrences */ def count(checkTrue: Boolean, start: Long, end: Long): Long @@ -97,14 +99,17 @@ trait LBitArrayOps { } /** + * Specialized implementaiton of LArray[Boolean] using LArray[Long] To generate an instance of LBitArray, use + * ``LBitArray.newBuilder(Long)`` or [[xerial.larray.LBitArray#apply]] * - * Specialized implementaiton of LArray[Boolean] using LArray[Long] - * To generate an instance of LBitArray, use ``LBitArray.newBuilder(Long)`` or [[xerial.larray.LBitArray#apply]] - * - * @param seq raw bit string + * @param seq + * raw bit string * @param numBits */ -class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) extends LArray[Boolean] with UnsafeArray[Boolean] with LBitArrayOps { +class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) + extends LArray[Boolean] + with UnsafeArray[Boolean] + with LBitArrayOps { self => @@ -116,29 +121,28 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) def this(numBits: Long) = this(new LLongArray(BitEncoder.minArraySize(numBits)), numBits) def this(numBits: Long, m: Memory) = this(new LLongArray(BitEncoder.minArraySize(numBits), m), numBits) - def size = numBits + def size = numBits private var hash: Int = 0 override def byteLength = seq.byteLength protected[this] def newBuilder: LBuilder[Boolean, LBitArray] = new LBitArrayBuilder() - private[larray] def m: Memory = seq.m + private[larray] def m: Memory = seq.m override def toString = { val displaySize = math.min(500L, numBits) - val b = new StringBuilder + val b = new StringBuilder for (i <- 0L until displaySize) { b.append(if (self(i)) { "1" - } - else { + } else { "0" }) } b.result() } - def on(index: Long) = update(index, true) + def on(index: Long) = update(index, true) def off(index: Long) = update(index, false) /** @@ -147,7 +151,7 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) * @return */ def fill { - unsafe.setMemory(seq.m.address, byteLength, 0xFF.toByte) + unsafe.setMemory(seq.m.address, byteLength, 0xff.toByte) } /** @@ -157,19 +161,18 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) * @return */ def apply(index: Long): Boolean = { - val addr = blockAddr(index) + val addr = blockAddr(index) val offset = blockOffset(index) - val code = ((seq.getLong(addr) >>> offset) & 1L).toInt + val code = ((seq.getLong(addr) >>> offset) & 1L).toInt table(code) } def update(index: Long, v: Boolean): Boolean = { - val addr = blockAddr(index) + val addr = blockAddr(index) val offset = blockOffset(index) if (v) { seq.putLong(addr, seq.getLong(addr) | (1L << offset)) - } - else { + } else { seq.putLong(addr, seq.getLong(addr) & (~(1L << offset))) } v @@ -181,7 +184,7 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) override def hashCode() = { if (hash == 0) { - var h = numBits * 31L + var h = numBits * 31L var pos = 0L for (i <- (0L until numFilledBlocks)) { h += seq(pos) * 31L @@ -200,11 +203,10 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) case other: LBitArray => { if (this.size != other.size) { false - } - else { + } else { (0L until numFilledBlocks).find(i => this.seq(i) != other.seq(i)) match { case Some(x) => false - case None => this.lastBlock == other.lastBlock + case None => this.lastBlock == other.lastBlock } } } @@ -217,8 +219,7 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) val c = java.lang.Long.bitCount(v) if (checkTrue) { c - } - else { + } else { 64L - c } } @@ -226,22 +227,24 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) /** * Count the number of bits within the specified range [start, end) * - * @param checkTrue count true or false + * @param checkTrue + * count true or false * @param start * @param end - * @return the number of occurrences + * @return + * the number of occurrences */ def count(checkTrue: Boolean, start: Long, end: Long): Long = { - val sPos = blockIndex(start) + val sPos = blockIndex(start) val sOffset = blockOffset(start) - val ePos = blockIndex(end - 1L) + val ePos = blockIndex(end - 1L) val eOffset = blockOffset(end - 1L) - var count = 0L + var count = 0L var num0sInMaskedRegion = 0L - var pos = sPos + var pos = sPos while (pos <= ePos) { var mask: Long = ~0L if (pos == sPos) { @@ -254,7 +257,7 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) num0sInMaskedRegion += 31L - eOffset } // Applying bit mask changes all bits to 0s, so we need to subtract num0s - val v: Long = seq(pos) & mask + val v: Long = seq(pos) & mask val popCount = fastCount(v, checkTrue) count += popCount pos += 1 @@ -262,8 +265,7 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) if (checkTrue) { count - num0sInMaskedRegion - } - else { + } else { count } } @@ -281,32 +283,30 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) } val sliceLen = end - start - val newSeq = new LLongArray(minArraySize(sliceLen)) + val newSeq = new LLongArray(minArraySize(sliceLen)) newSeq.clear var i = 0L while (i < sliceLen) { - val sPos = blockIndex(start + i) + val sPos = blockIndex(start + i) val sOffset = blockOffset(start + i) - val dPos = blockIndex(i) + val dPos = blockIndex(i) val dOffset = blockOffset(i) var copyLen = 0L - var l = 0L - val v = seq(sPos) & (~0L << sOffset) + var l = 0L + val v = seq(sPos) & (~0L << sOffset) if (sOffset == dOffset) { // no shift copyLen = 64L l = v - } - else if (sOffset < dOffset) { + } else if (sOffset < dOffset) { // left shift val shiftLen = dOffset - sOffset copyLen = 64L - dOffset l = v << shiftLen - } - else { + } else { // right shift val shiftLen = sOffset - dOffset copyLen = 64L - sOffset @@ -337,14 +337,13 @@ class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) val fout = new FileOutputStream(f).getChannel try { // LBitArray need to record numBits - val b = new Array[Byte](8) + val b = new Array[Byte](8) val bb = ByteBuffer.wrap(b).putLong(numBits) bb.flip() fout.write(bb) fout.write(this.toDirectByteBuffer) f - } - finally + } finally fout.close } @@ -359,9 +358,9 @@ class LBitArrayBuilder extends LBuilder[Boolean, LBitArray] with LogSupport { def elementSize = throw new UnsupportedOperationException("elementSize of LBitArrayBuilder") - private var elems : LLongArray = _ - private var capacity: Long = 0L - private var numBits : Long = 0L + private var elems: LLongArray = _ + private var capacity: Long = 0L + private var numBits: Long = 0L protected def mkArray(size: Long): LLongArray = { val newArray = new LLongArray(minArraySize(size)) @@ -382,8 +381,7 @@ class LBitArrayBuilder extends LBuilder[Boolean, LBitArray] with LogSupport { if (capacity < size || capacity == 0L) { var newsize = if (capacity <= 1L) { 64L - } - else { + } else { capacity * factor } while (newsize < size) newsize *= factor @@ -411,12 +409,11 @@ class LBitArrayBuilder extends LBuilder[Boolean, LBitArray] with LogSupport { */ override def +=(v: Boolean): this.type = { ensureSize(numBits + 1) - val pos = blockIndex(numBits) + val pos = blockIndex(numBits) val offset = blockOffset(numBits) if (v) { elems(pos) |= (1L << offset) - } - else { + } else { elems(pos) &= ~(1L << offset) } numBits += 1 @@ -427,8 +424,7 @@ class LBitArrayBuilder extends LBuilder[Boolean, LBitArray] with LogSupport { val s = minArraySize(numBits) if (capacity != 0L && capacity == s) { new LBitArray(elems, numBits) - } - else { + } else { new LBitArray(mkArray(numBits), numBits) } } diff --git a/larray/src/main/scala/xerial/larray/LByteBuffer.scala b/larray/src/main/scala/xerial/larray/LByteBuffer.scala index 221ee1d..8c08a71 100644 --- a/larray/src/main/scala/xerial/larray/LByteBuffer.scala +++ b/larray/src/main/scala/xerial/larray/LByteBuffer.scala @@ -25,98 +25,98 @@ package xerial.larray import java.nio.{InvalidMarkException, Buffer} /** - * ByteBuffer interface of [[xerial.larray.LArray]] - * @author Taro L. Saito - */ -class LByteBuffer(buf:LByteArray, private var cursor:Long, private var bufLimit:Long) { - private var _mark : Long = -1L + * ByteBuffer interface of [[xerial.larray.LArray]] + * @author + * Taro L. Saito + */ +class LByteBuffer(buf: LByteArray, private var cursor: Long, private var bufLimit: Long) { + private var _mark: Long = -1L - def this(buf:LByteArray) = this(buf, 0L, buf.byteLength) + def this(buf: LByteArray) = this(buf, 0L, buf.byteLength) /** - * Return the LArray representation of this buffer - */ - def array : LByteArray = buf + * Return the LArray representation of this buffer + */ + def array: LByteArray = buf - def position : Long = cursor - def position(newPos:Long) : this.type = { cursor = newPos; this } - def limit : Long = buf.byteLength - def limit(newLimit:Long) : this.type = { bufLimit = newLimit; this } + def position: Long = cursor + def position(newPos: Long): this.type = { cursor = newPos; this } + def limit: Long = buf.byteLength + def limit(newLimit: Long): this.type = { bufLimit = newLimit; this } - def flip : this.type = { + def flip: this.type = { bufLimit = cursor cursor = 0L _mark = -1L this } - def rewind : this.type = { + def rewind: this.type = { cursor = 0L _mark = -1L this } - def remaining : Long = bufLimit - cursor - def hasRemaining : Boolean = cursor < bufLimit + def remaining: Long = bufLimit - cursor + def hasRemaining: Boolean = cursor < bufLimit - def clear() : this.type = { + def clear(): this.type = { cursor = 0L bufLimit = buf.length _mark = -1L this } - def mark : this.type = { + def mark: this.type = { _mark = cursor this } - def reset : this.type = { + def reset: this.type = { val m = _mark if (m < 0L) throw new InvalidMarkException cursor = m this } - def put(b:Array[Byte]) : this.type = put(b, 0, b.length) - def put(b:Array[Byte], offset:Int, len:Int) : this.type = { + def put(b: Array[Byte]): this.type = put(b, 0, b.length) + def put(b: Array[Byte], offset: Int, len: Int): this.type = { buf.readFromArray(b, offset, cursor, len) cursor += len this } - def put[A](b:LArray[A], offset:Long, len:Long) : this.type = { + def put[A](b: LArray[A], offset: Long, len: Long): this.type = { b.copyTo(offset, buf, cursor, len) cursor += len this } - def putBoolean(v:Boolean) : this.type = { buf.putByte(cursor, if(v) 1 else 0); cursor += 1; this } - def putByte(v:Byte) : this.type = { buf.putByte(cursor, v); cursor += 1; this } - def putChar(v:Char) : this.type = { buf.putChar(cursor, v); cursor += 2; this } - def putShort(v:Short) : this.type = { buf.putShort(cursor, v); cursor += 2; this } - def putInt(v:Int) : this.type = { buf.putInt(cursor, v); cursor += 4; this } - def putFloat(v:Float) : this.type = { buf.putFloat(cursor, v); cursor += 4; this } - def putLong(v:Long) : this.type = { buf.putLong(cursor, v); cursor += 8; this } - def putDouble(v:Double) : this.type = { buf.putDouble(cursor, v); cursor += 8; this } - - def get(b:Array[Byte]) : this.type = get(b, 0, b.length) - def get(b:Array[Byte], offset:Int, len:Int) : this.type = { + def putBoolean(v: Boolean): this.type = { buf.putByte(cursor, if (v) 1 else 0); cursor += 1; this } + def putByte(v: Byte): this.type = { buf.putByte(cursor, v); cursor += 1; this } + def putChar(v: Char): this.type = { buf.putChar(cursor, v); cursor += 2; this } + def putShort(v: Short): this.type = { buf.putShort(cursor, v); cursor += 2; this } + def putInt(v: Int): this.type = { buf.putInt(cursor, v); cursor += 4; this } + def putFloat(v: Float): this.type = { buf.putFloat(cursor, v); cursor += 4; this } + def putLong(v: Long): this.type = { buf.putLong(cursor, v); cursor += 8; this } + def putDouble(v: Double): this.type = { buf.putDouble(cursor, v); cursor += 8; this } + + def get(b: Array[Byte]): this.type = get(b, 0, b.length) + def get(b: Array[Byte], offset: Int, len: Int): this.type = { buf.writeToArray(cursor, b, offset, len) this } - def get[A](b:RawByteArray[A], offset:Long, len:Long) : this.type = { + def get[A](b: RawByteArray[A], offset: Long, len: Long): this.type = { buf.copyTo(cursor, b, offset, len) cursor += len this } - def getBoolean: Boolean = { val v = if(buf.getByte(cursor) == 0) false else true; cursor += 1; v} - def getByte : Byte = { val v = buf.getByte(cursor); cursor += 1; v } - def getChar : Char = { val v = buf.getChar(cursor); cursor += 2; v } - def getShort : Short = { val v = buf.getShort(cursor); cursor += 2; v } - def getInt : Int = { val v = buf.getInt(cursor); cursor += 4; v } - def getFloat : Float = { val v = buf.getFloat(cursor); cursor += 4; v } - def getLong : Long = { val v = buf.getLong(cursor); cursor += 8; v } - def getDouble : Double = { val v = buf.getDouble(cursor); cursor += 8; v } - - -} \ No newline at end of file + def getBoolean: Boolean = { val v = if (buf.getByte(cursor) == 0) false else true; cursor += 1; v } + def getByte: Byte = { val v = buf.getByte(cursor); cursor += 1; v } + def getChar: Char = { val v = buf.getChar(cursor); cursor += 2; v } + def getShort: Short = { val v = buf.getShort(cursor); cursor += 2; v } + def getInt: Int = { val v = buf.getInt(cursor); cursor += 4; v } + def getFloat: Float = { val v = buf.getFloat(cursor); cursor += 4; v } + def getLong: Long = { val v = buf.getLong(cursor); cursor += 8; v } + def getDouble: Double = { val v = buf.getDouble(cursor); cursor += 8; v } + +} diff --git a/larray/src/main/scala/xerial/larray/LIterable.scala b/larray/src/main/scala/xerial/larray/LIterable.scala index ca4f38d..effe273 100644 --- a/larray/src/main/scala/xerial/larray/LIterable.scala +++ b/larray/src/main/scala/xerial/larray/LIterable.scala @@ -27,22 +27,22 @@ import annotation.tailrec import collection.{AbstractIterator, Iterator} import scala.inline - /** - * Iterable interface for LArray. - * - * @author Taro L. Saito - */ -trait LIterable[A] { self : LSeq[A] => + * Iterable interface for LArray. + * + * @author + * Taro L. Saito + */ +trait LIterable[A] { self: LSeq[A] => type Repr = LArray[A] /** - * Create a new array that concatenates two arrays - * @param other - * @return - */ - def concat(other:LSeq[A]) : Repr = { + * Create a new array that concatenates two arrays + * @param other + * @return + */ + def concat(other: LSeq[A]): Repr = { val b = newBuilder b.sizeHint(this.size + other.size) b.append(self) @@ -51,39 +51,48 @@ trait LIterable[A] { self : LSeq[A] => } /** - * Create a new array that concatenates two arrays - * @param other - * @return - */ - def ++(other:LSeq[A]) : Repr = concat(other) + * Create a new array that concatenates two arrays + * @param other + * @return + */ + def ++(other: LSeq[A]): Repr = concat(other) /** - * fold left - * @param z the start value - * @param op the binary operator - * @tparam B the result type of the binary operator - * @return the result of inserting op between consecutive elements of this array, going left to right with the start value z on the left: - * {{{ op(...op(op(z, x1), x2), ..., xn))) }}} - */ - def /:[B](z:B)(op:(B, A) => B) : B = foldLeft(z)(op) + * fold left + * @param z + * the start value + * @param op + * the binary operator + * @tparam B + * the result type of the binary operator + * @return + * the result of inserting op between consecutive elements of this array, going left to right with the start value + * z on the left: {{{op(...op(op(z, x1), x2), ..., xn)))}}} + */ + def /:[B](z: B)(op: (B, A) => B): B = foldLeft(z)(op) /** - * fold right - * @param z the start value - * @param op the binary operator - * @tparam B the result type of the binary operator - * @return the result of inserting op between consecutive elements of this array, going right to left with the start value z on the right: - * {{{ op(x1, op(x2, ..., op(xn, z)...)) }}} - * - */ - def :\[B](z:B)(op:(A, B) => B) : B = foldRight(z)(op) + * fold right + * @param z + * the start value + * @param op + * the binary operator + * @tparam B + * the result type of the binary operator + * @return + * the result of inserting op between consecutive elements of this array, going right to left with the start value + * z on the right: {{{op(x1, op(x2, ..., op(xn, z)...))}}} + */ + def :\[B](z: B)(op: (A, B) => B): B = foldRight(z)(op) /** - * Copy of this array with an element appended. - * @param elem the appended element - * @return a new array consisting of all elements of this array follwed by the new elem - */ - def :+(elem:A) : Repr = { + * Copy of this array with an element appended. + * @param elem + * the appended element + * @return + * a new array consisting of all elements of this array follwed by the new elem + */ + def :+(elem: A): Repr = { val b = newBuilder b.sizeHint(size + 1) b.append(self) @@ -92,11 +101,13 @@ trait LIterable[A] { self : LSeq[A] => } /** - * Copy of thie array with an element prepended. - * @param elem the prepended element. - * @return a new array consisting ofall elements of this array preceded by the new elem. - */ - def +:(elem:A) : Repr = { + * Copy of thie array with an element prepended. + * @param elem + * the prepended element. + * @return + * a new array consisting ofall elements of this array preceded by the new elem. + */ + def +:(elem: A): Repr = { val b = newBuilder b.sizeHint(size + 1) b.append(elem) @@ -105,12 +116,12 @@ trait LIterable[A] { self : LSeq[A] => } /** - * Provides the Iterable interface for Java - * @return - */ - def ji : java.lang.Iterable[A] = new java.lang.Iterable[A] { + * Provides the Iterable interface for Java + * @return + */ + def ji: java.lang.Iterable[A] = new java.lang.Iterable[A] { def iterator(): java.util.Iterator[A] = new java.util.Iterator[A] { - private var index = 0L + private var index = 0L def hasNext: Boolean = index < size def next(): A = { val v = self(index) @@ -121,14 +132,14 @@ trait LIterable[A] { self : LSeq[A] => } } - protected[this] def newBuilder : LBuilder[A, Repr] + protected[this] def newBuilder: LBuilder[A, Repr] /** - * Creates a new iterator over all elements contained in this collection - */ - def iterator : LIterator[A] = new AbstractLIterator[A] { - private var index = 0L - override def size = self.size + * Creates a new iterator over all elements contained in this collection + */ + def iterator: LIterator[A] = new AbstractLIterator[A] { + private var index = 0L + override def size = self.size def hasNext: Boolean = index < size def next(): A = { val v = self(index) @@ -138,7 +149,7 @@ trait LIterable[A] { self : LSeq[A] => } def reverseIterator: LIterator[A] = new AbstractLIterator[A] { - private var i = self.size + private var i = self.size def hasNext: Boolean = 0L < i def next(): A = if (0L < i) { @@ -147,37 +158,43 @@ trait LIterable[A] { self : LSeq[A] => } else LIterator.empty.next() } - def toIterator : LIterator[A] = iterator + def toIterator: LIterator[A] = iterator /** - * Creates a copy of this array in the form of the standard Scala Array - * @tparam A1 - * @return - */ - def toArray[A1 >: A : ClassTag] : Array[A1] = { + * Creates a copy of this array in the form of the standard Scala Array + * @tparam A1 + * @return + */ + def toArray[A1 >: A: ClassTag]: Array[A1] = { val b = Array.newBuilder[A1] foreach(b += _) b.result() } /** - * Tests whether this sequence is empty - * @return - */ - def isEmpty : Boolean = { size == 0L } + * Tests whether this sequence is empty + * @return + */ + def isEmpty: Boolean = { size == 0L } /** - * Builds a new collection by applying a partial function to all elments of this array on which the function is defined. - */ - def collect[B](pf:PartialFunction[A, B]) : LIterator[B] = iterator.collect(pf) + * Builds a new collection by applying a partial function to all elments of this array on which the function is + * defined. + */ + def collect[B](pf: PartialFunction[A, B]): LIterator[B] = iterator.collect(pf) /** - * Finds the first element of this array on which the given partial function is defined, and applies the partial function to it. - * @param pf partial function - * @tparam B return type - * @return an option value containing pf applied to the first value for which the function is defined, or None if not exists. - */ - def collectFirst[B](pf:PartialFunction[A, B]) : Option[B] = { + * Finds the first element of this array on which the given partial function is defined, and applies the partial + * function to it. + * @param pf + * partial function + * @tparam B + * return type + * @return + * an option value containing pf applied to the first value for which the function is defined, or None if not + * exists. + */ + def collectFirst[B](pf: PartialFunction[A, B]): Option[B] = { for (x <- self.toIterator) { // make sure to use an iterator or `seq` if (pf isDefinedAt x) return Some(pf(x)) @@ -185,49 +202,48 @@ trait LIterable[A] { self : LSeq[A] => None } - def contains(elem: A): Boolean = exists(_ == elem) - def exists(p: A => Boolean) : Boolean = iterator.exists(p) + def contains(elem: A): Boolean = exists(_ == elem) + def exists(p: A => Boolean): Boolean = iterator.exists(p) def find(p: A => Boolean): Option[A] = { val i = prefixLength(!p(_)) if (i < length) Some(this(i)) else None } - def filter(pred:A=>Boolean) : LIterator[A] = iterator.filter(pred) - def filterNot(pred:A=>Boolean) : LIterator[A] = iterator.filterNot(pred) - def forall(p: A => Boolean) : Boolean = prefixLength(p(_)) == length + def filter(pred: A => Boolean): LIterator[A] = iterator.filter(pred) + def filterNot(pred: A => Boolean): LIterator[A] = iterator.filterNot(pred) + def forall(p: A => Boolean): Boolean = prefixLength(p(_)) == length def foreach[U](f: A => U) { - var i = 0L + var i = 0L val len = size - while(i < len) { f(self(i)); i += 1} + while (i < len) { f(self(i)); i += 1 } } def prefixLength(p: A => Boolean): Long = segmentLength(p, 0L) def segmentLength(p: A => Boolean, from: Long): Long = { val len = length - var i = from + var i = from while (i < len && p(self(i))) i += 1 i - from } - def length : Long = size + def length: Long = size - def withFilter(p: A=>Boolean) : LIterator[A] = iterator.filter(p) + def withFilter(p: A => Boolean): LIterator[A] = iterator.filter(p) - def map[B](f:A=>B): LIterator[B] = iterator.map(f) - def flatMap[B](f: A => LIterator[B]) : LIterator[B] = iterator.flatMap(f) + def map[B](f: A => B): LIterator[B] = iterator.map(f) + def flatMap[B](f: A => LIterator[B]): LIterator[B] = iterator.flatMap(f) def reverse[A]: Repr = { val b = newBuilder b.sizeHint(size) var i = length - while(0L < i) { + while (0L < i) { i -= 1 b += self(i) } b.result() } - @tailrec private def foldl[B](start: Long, end: Long, z: B, op: (B, A) => B): B = if (start == end) z @@ -256,7 +272,6 @@ trait LIterable[A] { self : LSeq[A] => else throw new UnsupportedOperationException("empry.reduceRight") - def reduceLeftOption[B >: A](op: (B, A) => B): Option[B] = if (isEmpty) None else Some(reduceLeft(op)) @@ -271,10 +286,7 @@ trait LIterable[A] { self : LSeq[A] => def aggregate[B](z: B)(seqop: (B, A) => B, combop: (B, B) => B): B = foldLeft(z)(seqop) - - - - def scanLeft[B](z:B)(op:(B, A) => B) : LIterator[B] = iterator.scanLeft(z)(op) + def scanLeft[B](z: B)(op: (B, A) => B): LIterator[B] = iterator.scanLeft(z)(op) private def negLength(n: Long) = if (n >= length) -1L else n @@ -287,16 +299,16 @@ trait LIterable[A] { self : LSeq[A] => while (i >= 0 && !p(this(i))) i -= 1 i } - def indexOf[B>:A](elem:B) : Long = iterator.indexOf(elem) + def indexOf[B >: A](elem: B): Long = iterator.indexOf(elem) - def slice(from:Long) : LArray[A] = slice(from, size) + def slice(from: Long): LArray[A] = slice(from, size) - def slice(from:Long, until:Long) : LArray[A] = { + def slice(from: Long, until: Long): LArray[A] = { val lo = math.max(from, 0L) val hi = math.min(math.max(until, 0L), length) val elems = math.max(hi - lo, 0L) // Supply array size to reduce the number of memory allocation - val b = newBuilder + val b = newBuilder b.sizeHint(elems) var i = lo while (i < hi) { @@ -306,20 +318,20 @@ trait LIterable[A] { self : LSeq[A] => b.result() } - def head : A = if(isEmpty) throw new NoSuchElementException else this(0) - def tail : LArray[A] = if(isEmpty) throw new NoSuchElementException else slice(1L, size) - def last : A = if(length > 0) self(length - 1) else throw new NoSuchElementException + def head: A = if (isEmpty) throw new NoSuchElementException else this(0) + def tail: LArray[A] = if (isEmpty) throw new NoSuchElementException else slice(1L, size) + def last: A = if (length > 0) self(length - 1) else throw new NoSuchElementException - def drop(n:Long) : Repr = slice(n, length) + def drop(n: Long): Repr = slice(n, length) def init: Repr = if (length > 0L) slice(0L, length - 1L) else throw new UnsupportedOperationException("empty.init") - def take(n:Long) : Repr = slice(0L, n) - def takeRight(n: Long) : Repr = slice(length - n, length) - def takeWhile(p: A => Boolean) : Repr = take(prefixLength(p)) - def splitAt(n: Long) : (Repr, Repr) = (take(n), drop(n)) - def dropWhile(p: A => Boolean) : Repr = drop(prefixLength(p)) - def partition(p : A=>Boolean) = iterator.partition(p) - def span(p:A=>Boolean) : (Repr, Repr) = { - val l, r = newBuilder + def take(n: Long): Repr = slice(0L, n) + def takeRight(n: Long): Repr = slice(length - n, length) + def takeWhile(p: A => Boolean): Repr = take(prefixLength(p)) + def splitAt(n: Long): (Repr, Repr) = (take(n), drop(n)) + def dropWhile(p: A => Boolean): Repr = drop(prefixLength(p)) + def partition(p: A => Boolean) = iterator.partition(p) + def span(p: A => Boolean): (Repr, Repr) = { + val l, r = newBuilder var toLeft = true for (x <- this) { toLeft = toLeft && p(x) @@ -329,8 +341,8 @@ trait LIterable[A] { self : LSeq[A] => } def copyToArray[B >: A](xs: Array[B], start: Int, len: Int) { - var i = 0L - var j = start + var i = 0L + var j = start val end = length min len min (xs.length - start) while (i < end) { xs(j) = this(i) @@ -339,8 +351,8 @@ trait LIterable[A] { self : LSeq[A] => } } def copyToArray[B >: A](xs: LArray[B], start: Long, len: Long) { - var i = 0L - var j = start + var i = 0L + var j = start val end = length min len min (xs.length - start) while (i < end) { xs(j) = this(i) @@ -349,13 +361,12 @@ trait LIterable[A] { self : LSeq[A] => } } - def sameElements[B >: A](that: LIterable[B]): Boolean = iterator.sameElements(that.toIterator) def zipAll[B, A1 >: A, B1 >: B](that: LIterable[B], thisElem: A1, thatElem: B1): LIterator[(A1, B1)] = iterator.zipAll(that.toIterator, thisElem, thatElem) - def zipWithIndex : LIterator[(A, Long)] = iterator.zipWithIndex + def zipWithIndex: LIterator[(A, Long)] = iterator.zipWithIndex def zip[B](that: LIterable[B]): LIterator[(A, B)] = iterator.zip(that.toIterator) @@ -364,10 +375,9 @@ trait LIterable[A] { self : LSeq[A] => def mkString(start: String, sep: String, end: String): String = addString(new StringBuilder(), start, sep, end).toString() def mkString(sep: String): String = mkString("", sep, "") - def mkString: String = mkString("") + def mkString: String = mkString("") - - class SlidingIterator(size:Long, step:Long) extends AbstractLIterator[Repr] { + class SlidingIterator(size: Long, step: Long) extends AbstractLIterator[Repr] { require(size > 0 && step > 0, s"size:$size and step:$step must be greater than 0") private var cursor = 0L @@ -375,11 +385,11 @@ trait LIterable[A] { self : LSeq[A] => def next() = { val begin = cursor - val end = math.min(begin + size, self.size) - val b = newBuilder - b.sizeHint(end-begin) + val end = math.min(begin + size, self.size) + val b = newBuilder + b.sizeHint(end - begin) var i = begin - while(i < end) { + while (i < end) { b += self.apply(i) i += 1 } @@ -388,20 +398,24 @@ trait LIterable[A] { self : LSeq[A] => } } - /** - * Groups elements in fixed size blocks by passing a 'sliding window' over them - * @param size the number of elements per group - * @return An iterator producing group of elements. - */ - def sliding(size:Int) : LIterator[Repr] = sliding(size, 1) + * Groups elements in fixed size blocks by passing a 'sliding window' over them + * @param size + * the number of elements per group + * @return + * An iterator producing group of elements. + */ + def sliding(size: Int): LIterator[Repr] = sliding(size, 1) /** - * Groups elemnts in fixed size blocks by passing a 'sliding window' over them. - * @param size the number of elements per group - * @param step the distance between the first elements of successive groups - * @return An itertor producing group of elements. - */ - def sliding(size:Long, step:Long) : LIterator[Repr] = new SlidingIterator(size, step) - -} \ No newline at end of file + * Groups elemnts in fixed size blocks by passing a 'sliding window' over them. + * @param size + * the number of elements per group + * @param step + * the distance between the first elements of successive groups + * @return + * An itertor producing group of elements. + */ + def sliding(size: Long, step: Long): LIterator[Repr] = new SlidingIterator(size, step) + +} diff --git a/larray/src/main/scala/xerial/larray/LIterator.scala b/larray/src/main/scala/xerial/larray/LIterator.scala index cd90ecb..7f46446 100644 --- a/larray/src/main/scala/xerial/larray/LIterator.scala +++ b/larray/src/main/scala/xerial/larray/LIterator.scala @@ -30,29 +30,30 @@ import scala.Some import scala.Seq import collection.mutable.ArrayBuffer - /** - * Iterator for LArray. It is a extension of `scala.collection.Iterable` and most of the code is - * derived from its implementation except that the index type is Long instead of Int. - * @author Taro L. Saito - */ + * Iterator for LArray. It is a extension of `scala.collection.Iterable` and most of the code is derived from its + * implementation except that the index type is Long instead of Int. + * @author + * Taro L. Saito + */ trait LIterator[+A] { self => import LIterator._ - def hasNext : Boolean + def hasNext: Boolean def next(): A - /** Tests whether this iterator is empty. + /** + * Tests whether this iterator is empty. * - * @return `true` if hasNext is false, `false` otherwise. + * @return + * `true` if hasNext is false, `false` otherwise. */ def isEmpty: Boolean = !hasNext - def buffered: BufferedLIterator[A] = new AbstractLIterator[A] with BufferedLIterator[A] { - private var hd: A = _ + private var hd: A = _ private var hdDefined: Boolean = false def head: A = { @@ -74,19 +75,18 @@ trait LIterator[+A] { } - - def collect[B](pf:PartialFunction[A, B]) : LIterator[B] = { + def collect[B](pf: PartialFunction[A, B]): LIterator[B] = { val self = buffered new AbstractLIterator[B] { private def skip() { while (self.hasNext && !pf.isDefinedAt(self.head)) self.next() } def hasNext = { skip(); self.hasNext } - def next() = { skip(); pf(self.next()) } + def next() = { skip(); pf(self.next()) } } } def scanLeft[B](z: B)(op: (B, A) => B): LIterator[B] = new AbstractLIterator[B] { var hasNext = true - var elem = z + var elem = z def next() = if (hasNext) { val res = elem if (self.hasNext) elem = op(elem, self.next()) @@ -100,8 +100,7 @@ trait LIterator[+A] { // toBuffer.scanRight(z)(op).iterator // } - - def exists(p: A => Boolean) : Boolean = { + def exists(p: A => Boolean): Boolean = { var res = false while (!res && hasNext) res = p(next()) res @@ -117,15 +116,17 @@ trait LIterator[+A] { res } - - /** Returns the index of the first produced value satisfying a predicate, or -1. + /** + * Returns the index of the first produced value satisfying a predicate, or -1. * - * @param p the predicate to test values - * @return the index of the first produced value satisfying `p`, - * or -1 if such an element does not exist until the end of the iterator is reached. + * @param p + * the predicate to test values + * @return + * the index of the first produced value satisfying `p`, or -1 if such an element does not exist until the end of + * the iterator is reached. */ def indexWhere(p: A => Boolean): Long = { - var i = 0L + var i = 0L var found = false while (!found && hasNext) { if (p(next())) { @@ -137,15 +138,17 @@ trait LIterator[+A] { if (found) i else -1L } - /** Returns the index of the first occurrence of the specified - * object in this iterable object. + /** + * Returns the index of the first occurrence of the specified object in this iterable object. * - * @param elem element to search for. - * @return the index of the first occurrence of `elem` in the values produced by this iterator, - * or -1 if such an element does not exist until the end of the iterator is reached. + * @param elem + * element to search for. + * @return + * the index of the first occurrence of `elem` in the values produced by this iterator, or -1 if such an element + * does not exist until the end of the iterator is reached. */ def indexOf[B >: A](elem: B): Long = { - var i = 0L + var i = 0L var found = false while (!found && hasNext) { if (next() == elem) { @@ -157,60 +160,56 @@ trait LIterator[+A] { if (found) i else -1L } - - - def foreach[U](f: A => U) { while(hasNext) { f(next()) } } - def forall(pred: A => Boolean) : Boolean = { + def foreach[U](f: A => U) { while (hasNext) { f(next()) } } + def forall(pred: A => Boolean): Boolean = { var result = true - while(result && hasNext) result = pred(next()) + while (result && hasNext) result = pred(next()) result } - def map[B](f:A=>B): LIterator[B] = new AbstractLIterator[B] { - def next(): B = f(self.next()) + def map[B](f: A => B): LIterator[B] = new AbstractLIterator[B] { + def next(): B = f(self.next()) def hasNext: Boolean = self.hasNext } - def flatMap[B](f: A => LIterator[B]) : LIterator[B] = new AbstractLIterator[B] { - private var current : LIterator[B] = empty + def flatMap[B](f: A => LIterator[B]): LIterator[B] = new AbstractLIterator[B] { + private var current: LIterator[B] = empty def hasNext: Boolean = current.hasNext || self.hasNext && { current = f(self.next()); hasNext } - def next(): B = (if(hasNext) current else empty).next() + def next(): B = (if (hasNext) current else empty).next() } - def reduceLeft[B >: A](op: (B, A) => B): B = { if (isEmpty) throw new UnsupportedOperationException("empty.reduceLeft") - var first = true + var first = true var acc: B = 0.asInstanceOf[B] for (x <- self) { if (first) { acc = x first = false - } - else acc = op(acc, x) + } else acc = op(acc, x) } acc } - - def filter(pred:A=>Boolean) : LIterator[A] = new AbstractLIterator[A] { - private var head : A = _ - private var headDefined : Boolean = false - def next(): A = if(hasNext) { headDefined = false; head } else empty.next() + def filter(pred: A => Boolean): LIterator[A] = new AbstractLIterator[A] { + private var head: A = _ + private var headDefined: Boolean = false + def next(): A = if (hasNext) { headDefined = false; head } + else empty.next() def hasNext: Boolean = headDefined || { do { - if(!self.hasNext) return false + if (!self.hasNext) return false head = self.next() } while (!pred(head)) headDefined = true true } } - def filterNot(p: A => Boolean) : LIterator[A] = filter(!p(_)) + def filterNot(p: A => Boolean): LIterator[A] = filter(!p(_)) def addString(b: StringBuilder, start: String, sep: String, end: String): StringBuilder = { var first = true @@ -220,8 +219,7 @@ trait LIterator[+A] { if (first) { b append x first = false - } - else { + } else { b append sep b append x } @@ -235,12 +233,11 @@ trait LIterator[+A] { addString(new StringBuilder(), start, sep, end).toString() def mkString(sep: String): String = mkString("", sep, "") - def mkString: String = mkString("") + def mkString: String = mkString("") def withFilter(p: A => Boolean): LIterator[A] = filter(p) - - def sameElements(that:LIterator[_]) : Boolean = { + def sameElements(that: LIterator[_]): Boolean = { while (hasNext && that.hasNext) if (next() != that.next()) return false @@ -248,8 +245,8 @@ trait LIterator[+A] { !hasNext && !that.hasNext } - def slice(from:Long, until:Long) : LIterator[A] = { - val lo = from max 0 + def slice(from: Long, until: Long): LIterator[A] = { + val lo = from max 0 var toDrop = lo while (toDrop > 0 && self.hasNext) { self.next() @@ -258,41 +255,43 @@ trait LIterator[+A] { new AbstractLIterator[A] { private var remaining = until - lo - def hasNext = remaining > 0L && self.hasNext + def hasNext = remaining > 0L && self.hasNext def next(): A = if (remaining > 0) { remaining -= 1 self.next() - } - else empty.next() + } else empty.next() } } - def size : Long = { + def size: Long = { var count = 0L - for(x <- self) count += 1 + for (x <- self) count += 1 count } - - /** Selects first ''n'' values of this iterator. + /** + * Selects first ''n'' values of this iterator. * - * @param n the number of values to take - * @return an iterator producing only of the first `n` values of this iterator, or else the - * whole iterator, if it produces fewer than `n` values. + * @param n + * the number of values to take + * @return + * an iterator producing only of the first `n` values of this iterator, or else the whole iterator, if it produces + * fewer than `n` values. */ def take(n: Long): LIterator[A] = slice(0L, n) - - /** Takes longest prefix of values produced by this iterator that satisfy a predicate. + /** + * Takes longest prefix of values produced by this iterator that satisfy a predicate. * - * @param p The predicate used to test elements. - * @return An iterator returning the values produced by this iterator, until - * this iterator produces a value that does not satisfy - * the predicate `p`. + * @param p + * The predicate used to test elements. + * @return + * An iterator returning the values produced by this iterator, until this iterator produces a value that does not + * satisfy the predicate `p`. */ def takeWhile(p: A => Boolean): LIterator[A] = new AbstractLIterator[A] { - private var hd: A = _ + private var hd: A = _ private var hdDefined: Boolean = false private var tail: LIterator[A] = self @@ -302,23 +301,24 @@ trait LIterator[+A] { else tail = empty hdDefined } - def next() = if (hasNext) { hdDefined = false; hd } else empty.next() + def next() = if (hasNext) { hdDefined = false; hd } + else empty.next() } - - /** Partitions this iterator in two iterators according to a predicate. + /** + * Partitions this iterator in two iterators according to a predicate. * - * @param p the predicate on which to partition - * @return a pair of iterators: the iterator that satisfies the predicate - * `p` and the iterator that does not. - * The relative order of the elements in the resulting iterators - * is the same as in the original iterator. + * @param p + * the predicate on which to partition + * @return + * a pair of iterators: the iterator that satisfies the predicate `p` and the iterator that does not. The relative + * order of the elements in the resulting iterators is the same as in the original iterator. */ def partition(p: A => Boolean): (LIterator[A], LIterator[A]) = { val self = buffered class PartitionIterator(p: A => Boolean) extends AbstractLIterator[A] { var other: PartitionIterator = _ - val lookahead = new mutable.Queue[A] + val lookahead = new mutable.Queue[A] def skip() { while (self.hasNext && !p(self.head)) { other.lookahead += self.next() @@ -335,24 +335,26 @@ trait LIterator[+A] { (l, r) } - /** Splits this Iterator into a prefix/suffix pair according to a predicate. + /** + * Splits this Iterator into a prefix/suffix pair according to a predicate. * - * @param p the test predicate - * @return a pair of Iterators consisting of the longest prefix of this - * whose elements all satisfy `p`, and the rest of the Iterator. + * @param p + * the test predicate + * @return + * a pair of Iterators consisting of the longest prefix of this whose elements all satisfy `p`, and the rest of the + * Iterator. */ def span(p: A => Boolean): (LIterator[A], LIterator[A]) = { val self = buffered /** - * Giving a name to following iterator (as opposed to trailing) because - * anonymous class is represented as a structural type that trailing - * iterator is referring (the finish() method) and thus triggering - * handling of structural calls. It's not what's intended here. - */ + * Giving a name to following iterator (as opposed to trailing) because anonymous class is represented as a + * structural type that trailing iterator is referring (the finish() method) and thus triggering handling of + * structural calls. It's not what's intended here. + */ class Leading extends AbstractLIterator[A] { private var isDone = false - val lookahead = new mutable.Queue[A] + val lookahead = new mutable.Queue[A] def advance() = { self.hasNext && p(self.head) && { lookahead += self.next() @@ -377,45 +379,45 @@ trait LIterator[+A] { leading.finish() self } - def hasNext = it.hasNext - def next() = it.next() + def hasNext = it.hasNext + def next() = it.next() override def toString = "unknown-if-empty iterator" } (leading, trailing) } - - /** Advances this iterator past the first ''n'' elements, or the length of the iterator, whichever is smaller. + /** + * Advances this iterator past the first ''n'' elements, or the length of the iterator, whichever is smaller. * - * @param n the number of elements to drop - * @return an iterator which produces all values of the current iterator, except - * it omits the first `n` values. + * @param n + * the number of elements to drop + * @return + * an iterator which produces all values of the current iterator, except it omits the first `n` values. */ def drop(n: Long): LIterator[A] = slice(n, Long.MaxValue) - - def toArray[A1 >: A : ClassTag] : Array[A1] = { + def toArray[A1 >: A: ClassTag]: Array[A1] = { val b = Array.newBuilder[A1] foreach(b += _) b.result() } + def zipAll[B, A1 >: A, B1 >: B](that: LIterator[B], thisElem: A1, thatElem: B1): LIterator[(A1, B1)] = + new AbstractLIterator[(A1, B1)] { + def hasNext = self.hasNext || that.hasNext + def next(): (A1, B1) = + if (self.hasNext) { + if (that.hasNext) (self.next(), that.next()) + else (self.next(), thatElem) + } else { + if (that.hasNext) (thisElem, that.next()) + else empty.next() + } + } - def zipAll[B, A1 >: A, B1 >: B](that: LIterator[B], thisElem: A1, thatElem: B1): LIterator[(A1, B1)] = new AbstractLIterator[(A1, B1)] { - def hasNext = self.hasNext || that.hasNext - def next(): (A1, B1) = - if (self.hasNext) { - if (that.hasNext) (self.next(), that.next()) - else (self.next(), thatElem) - } else { - if (that.hasNext) (thisElem, that.next()) - else empty.next() - } - } - - def zipWithIndex : LIterator[(A, Long)] = new AbstractLIterator[(A, Long)] { + def zipWithIndex: LIterator[(A, Long)] = new AbstractLIterator[(A, Long)] { private var index = 0L def next(): (A, Long) = { val v = (self.next(), index) @@ -427,23 +429,21 @@ trait LIterator[+A] { def zip[B](that: LIterator[B]): LIterator[(A, B)] = new AbstractLIterator[(A, B)] { def hasNext = self.hasNext && that.hasNext - def next = (self.next, that.next) + def next = (self.next, that.next) } - - def toLArray[B >: A : ClassTag] : LArray[B] = { + def toLArray[B >: A: ClassTag]: LArray[B] = { val b = LArray.newBuilder[B] self.foreach(b += _) b.result } - } object LIterator { - val empty : LIterator[Nothing] = new AbstractLIterator[Nothing] { + val empty: LIterator[Nothing] = new AbstractLIterator[Nothing] { def hasNext: Boolean = false - def next(): Nothing = throw new NoSuchElementException("next on empty iterator") + def next(): Nothing = throw new NoSuchElementException("next on empty iterator") } } @@ -452,5 +452,4 @@ trait BufferedLIterator[+A] extends LIterator[A] { def head: A } - private[larray] abstract class AbstractLIterator[A] extends LIterator[A] diff --git a/larray/src/main/scala/xerial/larray/MappedLByteArray.scala b/larray/src/main/scala/xerial/larray/MappedLByteArray.scala index 37757a9..340ebc3 100644 --- a/larray/src/main/scala/xerial/larray/MappedLByteArray.scala +++ b/larray/src/main/scala/xerial/larray/MappedLByteArray.scala @@ -25,66 +25,74 @@ package xerial.larray import java.io.File import xerial.larray.buffer.{Memory, MemoryAllocator} -import xerial.larray.mmap.{MMapBuffer, MMapMode} +//import xerial.larray.mmap.{MMapBuffer, MMapMode} /** - * Memory-mapped LByteArray + * Memory-mapped LByteArray TODO: Java17 support * - * @author Taro L. Saito + * @author + * Taro L. Saito */ -class MappedLByteArray(f: File, offset: Long = 0, val size: Long = -1, mode: MMapMode = MMapMode.READ_WRITE)(implicit alloc: MemoryAllocator) extends RawByteArray[Byte] { - - import java.{lang => jl} - - import UnsafeUtil.unsafe - - private val mmap = new MMapBuffer(f, offset, size, mode); - private val m: Memory = mmap.m - - protected[this] def newBuilder = new LByteArrayBuilder - - val address = mmap.address() - - def free { - m.release(); - } - - /** - * Forces any changes made to this buffer to be written to the file - */ - def flush { - mmap.flush() - } - - /** - * Close the memory mapped file. To ensure the written data is saved in the file, call flush before closing - */ - override def close() { - mmap.close() - } - - /** - * Update an element - * - * @param i index to be updated - * @param v value to set - * @return the value - */ - def update(i: Long, v: Byte) = {unsafe.putByte(address + i, v); v} - - def view(from: Long, to: Long) = new LArrayView.LByteArrayView(this, from, to - from) - - /** - * Retrieve an element - * - * @param i index - * @return the element value - */ - def apply(i: Long) = unsafe.getByte(address + i) - - /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ - private[larray] def elementByteSize = 1 - -} \ No newline at end of file +//class MappedLByteArray(f: File, offset: Long = 0, val size: Long = -1, mode: MMapMode = MMapMode.READ_WRITE)(implicit +// alloc: MemoryAllocator +//) extends RawByteArray[Byte] { +// +// import java.{lang => jl} +// +// import UnsafeUtil.unsafe +// +// private val mmap = new MMapBuffer(f, offset, size, mode); +// private val m: Memory = mmap.m +// +// protected[this] def newBuilder = new LByteArrayBuilder +// +// val address = mmap.address() +// +// def free { +// m.release(); +// } +// +// /** +// * Forces any changes made to this buffer to be written to the file +// */ +// def flush { +// mmap.flush() +// } +// +// /** +// * Close the memory mapped file. To ensure the written data is saved in the file, call flush before closing +// */ +// override def close() { +// mmap.close() +// } +// +// /** +// * Update an element +// * +// * @param i +// * index to be updated +// * @param v +// * value to set +// * @return +// * the value +// */ +// def update(i: Long, v: Byte) = { unsafe.putByte(address + i, v); v } +// +// def view(from: Long, to: Long) = new LArrayView.LByteArrayView(this, from, to - from) +// +// /** +// * Retrieve an element +// * +// * @param i +// * index +// * @return +// * the element value +// */ +// def apply(i: Long) = unsafe.getByte(address + i) +// +// /** +// * Byte size of an element. For example, if A is Int, its elementByteSize is 4 +// */ +// private[larray] def elementByteSize = 1 +// +//} diff --git a/larray/src/main/scala/xerial/larray/UInt32Array.scala b/larray/src/main/scala/xerial/larray/UInt32Array.scala index 7add169..f4640bd 100644 --- a/larray/src/main/scala/xerial/larray/UInt32Array.scala +++ b/larray/src/main/scala/xerial/larray/UInt32Array.scala @@ -30,68 +30,69 @@ object UInt32Array { def +=(v: Long): this.type = { ensureSize(numElems + 1) - elems.putInt(cursor, (v & 0xFFFFFFFFL).toInt) + elems.putInt(cursor, (v & 0xffffffffL).toInt) cursor += elementSize this } - /** Produces a collection from the added elements. - * The builder's contents are undefined after this operation. - * @return a collection containing the elements added to this builder. + /** + * Produces a collection from the added elements. The builder's contents are undefined after this operation. + * @return + * a collection containing the elements added to this builder. */ def result(): UInt32Array = { - if(capacity != 0L && capacity == numElems) new UInt32Array(numElems, elems.m) + if (capacity != 0L && capacity == numElems) new UInt32Array(numElems, elems.m) else new UInt32Array(numElems, mkArray(numElems).m) } def elementSize = 4 } - - } - -private[larray] class UInt32ArrayView(base:UInt32Array, offset:Long, val size:Long) extends LArrayView[Long] { +private[larray] class UInt32ArrayView(base: UInt32Array, offset: Long, val size: Long) extends LArrayView[Long] { protected[this] def newBuilder: LBuilder[Long, UInt32Array] = UInt32Array.newBuilder - def apply(i: Long) = base.apply(offset + i) - private[larray] def elementByteSize = 4 + def apply(i: Long) = base.apply(offset + i) + private[larray] def elementByteSize = 4 def copyTo(dst: LByteArray, dstOffset: Long) { base.copyTo(offset, dst, dstOffset, byteLength) } - def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { base.copyTo(offset+srcOffset, dst, dstOffset, blen) } + def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { + base.copyTo(offset + srcOffset, dst, dstOffset, blen) + } def address = base.address + (offset * elementByteSize) } /** - * Array of uint32 values. The internal array representation is the same with LIntArray, but the apply and update methods are based on Long type values. - * - * @author Taro L. Saito - */ -class UInt32Array(val size: Long, private[larray] val m:Memory)(implicit val alloc: MemoryAllocator) extends LArray[Long] with UnsafeArray[Long] { self => - def this(size:Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) + * Array of uint32 values. The internal array representation is the same with LIntArray, but the apply and update + * methods are based on Long type values. + * + * @author + * Taro L. Saito + */ +class UInt32Array(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) + extends LArray[Long] + with UnsafeArray[Long] { self => + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) import UnsafeUtil.unsafe protected[this] def newBuilder: LBuilder[Long, UInt32Array] = UInt32Array.newBuilder - def apply(i:Long) : Long = { - val v : Long = unsafe.getInt(m.address + (i << 2)) & 0xFFFFFFFFL + def apply(i: Long): Long = { + val v: Long = unsafe.getInt(m.address + (i << 2)) & 0xffffffffL v } - def update(i:Long, v:Long) : Long = { - unsafe.putInt(m.address + (i << 2), (v & 0xFFFFFFFFL).toInt) + def update(i: Long, v: Long): Long = { + unsafe.putInt(m.address + (i << 2), (v & 0xffffffffL).toInt) v } /** - * Byte size of an element. For example, if A is Int, its elementByteSize is 4 - */ + * Byte size of an element. For example, if A is Int, its elementByteSize is 4 + */ private[larray] def elementByteSize: Int = 4 - - def view(from: Long, to: Long) : LArrayView[Long] = new UInt32ArrayView(self, from, to - from) + def view(from: Long, to: Long): LArrayView[Long] = new UInt32ArrayView(self, from, to - from) } - - diff --git a/larray/src/main/scala/xerial/larray/UnsafeUtil.scala b/larray/src/main/scala/xerial/larray/UnsafeUtil.scala index 1b0ddc1..77315ef 100644 --- a/larray/src/main/scala/xerial/larray/UnsafeUtil.scala +++ b/larray/src/main/scala/xerial/larray/UnsafeUtil.scala @@ -30,7 +30,8 @@ import wvlet.log.LogSupport /** * Utilities for accessing sun.misc.Unsafe * - * @author Taro L. Saito + * @author + * Taro L. Saito */ object UnsafeUtil extends LogSupport { val unsafe = { @@ -53,8 +54,7 @@ object UnsafeUtil extends LogSupport { val addressBandWidth = System.getProperty("sun.arch.data.model", "64").toInt private val addressFactor = if (addressBandWidth == 64) { 8L - } - else { + } else { 1L } val addressSize = unsafe.addressSize() @@ -62,17 +62,21 @@ object UnsafeUtil extends LogSupport { /** * @param obj * @return - * */ - @deprecated(message = "Deprecated because this method does not return correct object addresses in some platform", since = "0.1") + @deprecated( + message = "Deprecated because this method does not return correct object addresses in some platform", + since = "0.1" + ) def getObjectAddr(obj: AnyRef): Long = { - trace(f"address factor:$addressFactor%d, addressSize:$addressSize, objectArrayOffset:$objectArrayOffset, objectArrayScale:$objectArrayScale") + trace( + f"address factor:$addressFactor%d, addressSize:$addressSize, objectArrayOffset:$objectArrayOffset, objectArrayScale:$objectArrayScale" + ) val o = new Array[AnyRef](1) o(0) = obj objectArrayScale match { - case 4 => (unsafe.getInt(o, objectArrayOffset) & 0xFFFFFFFFL) * addressFactor - case 8 => (unsafe.getLong(o, objectArrayOffset) & 0xFFFFFFFFFFFFFFFFL) * addressFactor + case 4 => (unsafe.getInt(o, objectArrayOffset) & 0xffffffffL) * addressFactor + case 8 => (unsafe.getLong(o, objectArrayOffset) & 0xffffffffffffffffL) * addressFactor } } } diff --git a/larray/src/main/scala/xerial/larray/example/LArrayExample.scala b/larray/src/main/scala/xerial/larray/example/LArrayExample.scala index d0ee663..86afd7d 100644 --- a/larray/src/main/scala/xerial/larray/example/LArrayExample.scala +++ b/larray/src/main/scala/xerial/larray/example/LArrayExample.scala @@ -1,93 +1,93 @@ -/*-------------------------------------------------------------------------- - * Copyright 2013 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// -// LArrayExample.scala -// Since: 2013/03/23 23:16 -// -//-------------------------------------- - -package xerial.larray.example - -/** - * Example of LArray Usages - * @author Taro L. Saito - */ -class LArrayExample { - - import xerial.larray._ - - // Create a new LArray of Int type of length = 5 - val l = LArray.of[Int](5) - - // Create an LArray with initial values - val ll = LArray(3, 5, 9, 10) - - // Set elements. To specify a range of LArray, use 'Until' (for Long range) instead of 'until' (for Int range). - for(i <- 0 Until l.size) - l(i) = i.toInt - - // Read elements - val e0 = l(0) - val e1 = l(1) - - // Print the elements - println(l.mkString(", ")) // 0, 1, 2, 3, 4 - - // Traverse the elements with their indexes - for((e, i) <- l.zipWithIndex) - println(s"l($i) = $e") // l(0) = 0, l(1) = 1, ... - - // Manipulate LArray - val l2 = l.map(_ * 10) // LArray(0, 10, 20, 30, 40) - val f = l.filter(_ % 2 == 0) // LArray(0, 2, 4) - val s = l.slice(2) // LArray(2, 3, 4) - l.foreach(println(_)) - - // Build LArray - val b = LArray.newBuilder[Int] - for(i <- 0 until (10, step=3)) - b += i - val lb = b.result // LArray(0, 3, 6, 9) - - // Convert to Scala Array - val arr = l.toArray - println(arr.mkString(", ")) // 0, 1, 2, 3, 4 - - // Convert Scala Array to LArray - val arr2 = Array(1, 3, 5) - val la = arr2.toLArray - - // Save to a file - import java.io.File - val file = l.saveTo(new File("target/larray.tmp")) - file.deleteOnExit() - // Load from a file - val l3 = LArray.loadFrom[Int](file) // LArray(0, 1, 2, 3, 4) - - // Initialize the array - l.clear() - println(l.mkString(", ")) // 0, 0, 0, 0, 0 - - - // Release the memory contents. - l.free - l3.free - - // You can omit calling free, because GC collects unused LArrays - - println("done.") -} \ No newline at end of file +/*-------------------------------------------------------------------------- + * Copyright 2013 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// +// LArrayExample.scala +// Since: 2013/03/23 23:16 +// +//-------------------------------------- + +package xerial.larray.example + +/** + * Example of LArray Usages + * @author + * Taro L. Saito + */ +class LArrayExample { + + import xerial.larray._ + + // Create a new LArray of Int type of length = 5 + val l = LArray.of[Int](5) + + // Create an LArray with initial values + val ll = LArray(3, 5, 9, 10) + + // Set elements. To specify a range of LArray, use 'Until' (for Long range) instead of 'until' (for Int range). + for (i <- 0 Until l.size) + l(i) = i.toInt + + // Read elements + val e0 = l(0) + val e1 = l(1) + + // Print the elements + println(l.mkString(", ")) // 0, 1, 2, 3, 4 + + // Traverse the elements with their indexes + for ((e, i) <- l.zipWithIndex) + println(s"l($i) = $e") // l(0) = 0, l(1) = 1, ... + + // Manipulate LArray + val l2 = l.map(_ * 10) // LArray(0, 10, 20, 30, 40) + val f = l.filter(_ % 2 == 0) // LArray(0, 2, 4) + val s = l.slice(2) // LArray(2, 3, 4) + l.foreach(println(_)) + + // Build LArray + val b = LArray.newBuilder[Int] + for (i <- 0 until (10, step = 3)) + b += i + val lb = b.result // LArray(0, 3, 6, 9) + + // Convert to Scala Array + val arr = l.toArray + println(arr.mkString(", ")) // 0, 1, 2, 3, 4 + + // Convert Scala Array to LArray + val arr2 = Array(1, 3, 5) + val la = arr2.toLArray + + // Save to a file + import java.io.File + val file = l.saveTo(new File("target/larray.tmp")) + file.deleteOnExit() + // Load from a file + val l3 = LArray.loadFrom[Int](file) // LArray(0, 1, 2, 3, 4) + + // Initialize the array + l.clear() + println(l.mkString(", ")) // 0, 0, 0, 0, 0 + + // Release the memory contents. + l.free + l3.free + + // You can omit calling free, because GC collects unused LArrays + + println("done.") +} diff --git a/larray/src/main/scala/xerial/larray/package.scala b/larray/src/main/scala/xerial/larray/package.scala index c066a9a..aac75f8 100644 --- a/larray/src/main/scala/xerial/larray/package.scala +++ b/larray/src/main/scala/xerial/larray/package.scala @@ -19,43 +19,49 @@ import reflect.ClassTag import xerial.larray.buffer.LBufferConfig /** - * == LArray == - * [[xerial.larray.LArray]] is a large off-heap array that can hold more than 2G (2^31) entries. - * - * === Features === - * - * - Supporting huge array size upto 2^63 -1 entries. - * - 2^31 -1 (2G) is the limitation of the default Java/Scala array size, because 32-bit signed integer (int) is used for the array indexes. To resolve this, LArray uses long type indexes of 64-bit signed integers. - * - For example the entire human genome data (3GB) can be stored in LArray. - * - LArray can be released from the main memory immediately. - * - Call LArray.free to release acquired memory resources. - * - The default arrays in Java/Scala are resident in JVM heaps, in which users cannot free the allocate arrays space even if they become unnecessary. That means it is hard to avoid OutOfMemoryException when working with large amount of data. - * - LArray is in sync with Garbage Collection (GC) - * - Even if you forget to call LArray.free, the acquired memory will be released when GC sweeps LArray instances. - * - To prevent accidental memory release, keep a reference to LArray somewhere (e.g., in List) - * - LArray uses off-heap memory, so it is free from the limitation of JVM memory manager. - * - LArray uses memory space outside of the default JVM heap, so creating LArrays with more than -Xmx(maximum memory size) is possible. This is useful when you need large amount of memory or the amount of memory required in your application is unknown. - * - Fast copy and memory allocation - * - Rich set of operations for LArray[A] - * - map, filter, reduce, zip, etc. - * - See [[xerial.larray.LArray]] - * - * === Limitations === - * - * - LArray[A] of generic objects (e.g., LArray[String], LArray[AnyRef]) cannot be released immedeately from the main memory, because objects other than primitive types need to be created on JVM heaps and become subject to GC. - * - To release objects from main memory, you need to create *off-heap* objects. For example, create a large `LArray[Byte]`, then align your object data on the array. Object parameters can be retrieved with `LArray[Byte].getInt(offset)`, `getFloat(offset)`, etc. - * - */ + * ==LArray== + * [[xerial.larray.LArray]] is a large off-heap array that can hold more than 2G (2^31) entries. + * + * ===Features=== + * + * - Supporting huge array size upto 2^63 -1 entries. + * - 2^31 -1 (2G) is the limitation of the default Java/Scala array size, because 32-bit signed integer (int) is + * used for the array indexes. To resolve this, LArray uses long type indexes of 64-bit signed integers. + * - For example the entire human genome data (3GB) can be stored in LArray. + * - LArray can be released from the main memory immediately. + * - Call LArray.free to release acquired memory resources. + * - The default arrays in Java/Scala are resident in JVM heaps, in which users cannot free the allocate arrays + * space even if they become unnecessary. That means it is hard to avoid OutOfMemoryException when working with + * large amount of data. + * - LArray is in sync with Garbage Collection (GC) + * - Even if you forget to call LArray.free, the acquired memory will be released when GC sweeps LArray instances. + * - To prevent accidental memory release, keep a reference to LArray somewhere (e.g., in List) + * - LArray uses off-heap memory, so it is free from the limitation of JVM memory manager. + * - LArray uses memory space outside of the default JVM heap, so creating LArrays with more than -Xmx(maximum + * memory size) is possible. This is useful when you need large amount of memory or the amount of memory required + * in your application is unknown. + * - Fast copy and memory allocation + * - Rich set of operations for LArray[A] + * - map, filter, reduce, zip, etc. + * - See [[xerial.larray.LArray]] + * + * ===Limitations=== + * + * - LArray[A] of generic objects (e.g., LArray[String], LArray[AnyRef]) cannot be released immedeately from the main + * memory, because objects other than primitive types need to be created on JVM heaps and become subject to GC. + * - To release objects from main memory, you need to create *off-heap* objects. For example, create a large + * `LArray[Byte]`, then align your object data on the array. Object parameters can be retrieved with + * `LArray[Byte].getInt(offset)`, `getFloat(offset)`, etc. + */ package object larray { - implicit def defaultAllocator : xerial.larray.buffer.MemoryAllocator = LBufferConfig.allocator + implicit def defaultAllocator: xerial.larray.buffer.MemoryAllocator = LBufferConfig.allocator - - implicit class ConvertArrayToLArray[A : ClassTag](arr:Array[A]) { - def toLArray : LArray[A] = { + implicit class ConvertArrayToLArray[A: ClassTag](arr: Array[A]) { + def toLArray: LArray[A] = { val l = LArray.of[A](arr.length).asInstanceOf[RawByteArray[A]] var i = 0 - while(i < arr.length) { + while (i < arr.length) { l(i) = arr(i) i += 1 } @@ -63,19 +69,19 @@ package object larray { } } - implicit class ConvertIterableToLArray[A : ClassTag](it:Iterable[A]) { - def toLArray : LArray[A] = { + implicit class ConvertIterableToLArray[A: ClassTag](it: Iterable[A]) { + def toLArray: LArray[A] = { val b = LArray.newBuilder[A] it.foreach(b += _) - b.result + b.result() } } - implicit class AsLRange(from:Long) { - def Until(to:Long) : LIterator[Long] = Until(to, 1) - def Until(to:Long, step:Long) : LIterator[Long] = new AbstractLIterator[Long] { + implicit class AsLRange(from: Long) { + def Until(to: Long): LIterator[Long] = Until(to, 1) + def Until(to: Long, step: Long): LIterator[Long] = new AbstractLIterator[Long] { private var cursor = from - def hasNext = cursor < to + def hasNext = cursor < to def next() = { val v = cursor cursor += step @@ -83,12 +89,11 @@ package object larray { } } + def To(to: Long): LIterator[Long] = To(to, 1) - def To(to:Long) : LIterator[Long] = To(to, 1) - - def To(to:Long, step:Long) : LIterator[Long] = new AbstractLIterator[Long] { + def To(to: Long, step: Long): LIterator[Long] = new AbstractLIterator[Long] { private var cursor = from - def hasNext = cursor <= to + def hasNext = cursor <= to def next() = { val v = cursor cursor += step @@ -98,6 +103,4 @@ package object larray { } - - -} \ No newline at end of file +} diff --git a/larray/src/main/scala/xerial/larray/util/Logger.scala b/larray/src/main/scala/xerial/larray/util/Logger.scala index 022dd0e..b594f17 100644 --- a/larray/src/main/scala/xerial/larray/util/Logger.scala +++ b/larray/src/main/scala/xerial/larray/util/Logger.scala @@ -1,40 +1,41 @@ -/*-------------------------------------------------------------------------- - * Copyright 2013 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// -// Logger.scala -// Since: 2013/03/22 2:32 -// -//-------------------------------------- - -package xerial.larray.util - -/** - * Logger wrapper for using `wvlet.log.Logger` in Java - * - * @author Taro L. Saito - */ -class Logger(cl:Class[_]) { - private val _logger = wvlet.log.Logger.apply(cl.getName) - - def trace(m:String) { _logger.trace(m) } - def debug(m:String) { _logger.debug(m) } - def info(m:String) { _logger.info(m) } - def warn(m:String) { _logger.warn(m) } - def error(m:String) { _logger.error(m) } - def fatal(m:String) { _logger.error(m) } - -} \ No newline at end of file +/*-------------------------------------------------------------------------- + * Copyright 2013 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// +// Logger.scala +// Since: 2013/03/22 2:32 +// +//-------------------------------------- + +package xerial.larray.util + +/** + * Logger wrapper for using `wvlet.log.Logger` in Java + * + * @author + * Taro L. Saito + */ +class Logger(cl: Class[_]) { + private val _logger = wvlet.log.Logger.apply(cl.getName) + + def trace(m: String): Unit = { _logger.trace(m) } + def debug(m: String): Unit = { _logger.debug(m) } + def info(m: String): Unit = { _logger.info(m) } + def warn(m: String): Unit = { _logger.warn(m) } + def error(m: String): Unit = { _logger.error(m) } + def fatal(m: String): Unit = { _logger.error(m) } + +} diff --git a/larray/src/test/scala/xerial/larray/IOPerfTest.scala b/larray/src/test/scala/xerial/larray/IOPerfTest.scala index 2921a1c..260e94a 100644 --- a/larray/src/test/scala/xerial/larray/IOPerfTest.scala +++ b/larray/src/test/scala/xerial/larray/IOPerfTest.scala @@ -22,22 +22,24 @@ package xerial.larray -import java.io._ +import wvlet.airspec.AirSpec -import wvlet.log.io.IOUtil +import java.io._ +import wvlet.log.io.{IOUtil, Timer} import scala.util.Random /** - * @author Taro L. Saito + * @author + * Taro L. Saito */ -class IOPerfTest extends LArraySpec { +class IOPerfTest extends AirSpec with Timer { def createSampleFile: File = { val file = File.createTempFile("sample", ".larray", new File("target")) file.deleteOnExit() val b = new Array[Byte](1024 * 1024) - //val P = 1024 * 1024 + // val P = 1024 * 1024 val P = 64 val f = new FileOutputStream(file) for (i <- 0 until P) { @@ -48,8 +50,8 @@ class IOPerfTest extends LArraySpec { file } - "LArray" should { - "compare I/O performance" in { + test("LArray") { + test("compare I/O performance") { val f1 = createSampleFile time("read", repeat = 10) { block("LArray.loadFrom") { @@ -59,11 +61,11 @@ class IOPerfTest extends LArraySpec { } block("FileOutputStream") { trace("Loading to Array") - IOUtil.readFully(new BufferedInputStream(new FileInputStream(f1))) {buf => + IOUtil.readFully(new BufferedInputStream(new FileInputStream(f1))) { buf => // do nothing } } } } } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/LArrayBuilderTest.scala b/larray/src/test/scala/xerial/larray/LArrayBuilderTest.scala index f8bd367..ad4bcc3 100644 --- a/larray/src/test/scala/xerial/larray/LArrayBuilderTest.scala +++ b/larray/src/test/scala/xerial/larray/LArrayBuilderTest.scala @@ -22,49 +22,51 @@ package xerial.larray +import wvlet.airspec.AirSpec + import scala.util.Random /** - * @author Taro L. Saito - */ -class LArrayBuilderTest extends LArraySpec { + * @author + * Taro L. Saito + */ +class LArrayBuilderTest extends AirSpec { - "LArrayBuilder" should { + test("LArrayBuilder") { - "build LArray" in { + test("build LArray") { val b = LArray.newBuilder[Int] - def elem(i:Long) = math.toDegrees(math.sin(i / 15.0)).toInt + def elem(i: Long) = math.toDegrees(math.sin(i / 15.0)).toInt - for(i <- 0L until 100L) + for (i <- 0L until 100L) b += elem(i) val l = b.result debug(l.mkString(", ")) - l.size should be (100) - l.zipWithIndex.forall {case (v, i) => v == elem(i) } should be (true) + l.size shouldBe (100) + l.zipWithIndex.forall { case (v, i) => v == elem(i) } shouldBe (true) } - "build large LArray" in { + test("build large LArray") { val b = LArray.newBuilder[Byte] val N = 3L * 1024 * 1024 debug("Creating large array") val r = new Random(0) var i = 0L - while(i < N) { + while (i < N) { b += r.nextInt(255).toByte i += 1 } val l = b.result - l.size should be (N) + l.size shouldBe (N) debug("Checking the elements") val r2 = new Random(0) - l.forall(v => v == r2.nextInt(255).toByte) should be (true) + l.forall(v => v == r2.nextInt(255).toByte) shouldBe (true) } - } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/LArrayFunctionTest.scala b/larray/src/test/scala/xerial/larray/LArrayFunctionTest.scala index 147aac3..85c518b 100644 --- a/larray/src/test/scala/xerial/larray/LArrayFunctionTest.scala +++ b/larray/src/test/scala/xerial/larray/LArrayFunctionTest.scala @@ -23,8 +23,8 @@ package xerial.larray import java.io.File - import org.scalatest._ +import org.scalatest.matchers.should.Matchers import wvlet.log.LogSupport import scala.reflect.ClassTag @@ -32,27 +32,26 @@ import scala.reflect.ClassTag object LArrayFunctionTest extends LogSupport with Matchers { def stringRepr[A: ClassTag](l: LSeq[A]): String = { - val tag = implicitly[ClassTag[A]] + val tag = implicitly[ClassTag[A]] val isBoolean = (tag.runtimeClass == java.lang.Boolean.TYPE) if (isBoolean) { l.toString - } - else { + } else { l.mkString(", ") } } def stringRepr[A: ClassTag](l: Seq[A]): String = { - val tag = implicitly[ClassTag[A]] + val tag = implicitly[ClassTag[A]] val isBoolean = (tag.runtimeClass == java.lang.Boolean.TYPE) if (isBoolean) { - l.map(v => if (v.asInstanceOf[Boolean]) { - "1" - } - else { - "0" - }).mkString - } - else { + l.map(v => + if (v.asInstanceOf[Boolean]) { + "1" + } else { + "0" + } + ).mkString + } else { l.mkString(", ") } } @@ -88,7 +87,7 @@ trait LArrayBehaviour { import LArrayFunctionTest._ - def validArray[A: ClassTag](arr: Seq[A]) = { + def validArray[A: ClassTag](arr: Seq[A]): Unit = { val l: LArray[A] = arr.toLArray When(s"input is (${stringRepr(arr).take(100)})") @@ -225,9 +224,9 @@ trait LArrayBehaviour { "fold elements" in { if (arr.length <= 1000) { l.foldLeft(0d)(_ + _) shouldBe arr.foldLeft(0d)(_ + _) - (0d /: l) (_ + _) shouldBe ((0d /: arr) (_ + _)) + (0d /: l)(_ + _) shouldBe ((0d /: arr)(_ + _)) l.foldRight(0d)(_ + _) shouldBe arr.foldRight(0d)(_ + _) - (l :\ 0d) (_ + _) shouldBe (arr :\ 0d) (_ + _) + (l :\ 0d)(_ + _) shouldBe (arr :\ 0d) (_ + _) } } @@ -300,9 +299,9 @@ trait LArrayBehaviour { "fold elements" in { if (arr.length <= 1000) { l.foldLeft(0f)(_ + _) shouldBe arr.foldLeft(0f)(_ + _) - (0f /: l) (_ + _) shouldBe ((0f /: arr) (_ + _)) + (0f /: l)(_ + _) shouldBe ((0f /: arr)(_ + _)) l.foldRight(0f)(_ + _) shouldBe arr.foldRight(0f)(_ + _) - (l :\ 0f) (_ + _) shouldBe (arr :\ 0f) (_ + _) + (l :\ 0f)(_ + _) shouldBe (arr :\ 0f) (_ + _) } } @@ -376,9 +375,9 @@ trait LArrayBehaviour { "fold elements" in { if (arr.length <= 1000) { l.foldLeft(0)(_ + _) shouldBe arr.foldLeft(0)(_ + _) - (0 /: l) (_ + _) shouldBe ((0 /: arr) (_ + _)) + (0 /: l)(_ + _) shouldBe ((0 /: arr)(_ + _)) l.foldRight(0)(_ + _) shouldBe arr.foldRight(0)(_ + _) - (l :\ 0) (_ + _) shouldBe (arr :\ 0) (_ + _) + (l :\ 0)(_ + _) shouldBe (arr :\ 0) (_ + _) } } @@ -452,9 +451,9 @@ trait LArrayBehaviour { "fold elements" in { if (arr.length <= 1000) { l.foldLeft(0L)(_ + _) shouldBe arr.foldLeft(0L)(_ + _) - (0L /: l) (_ + _) shouldBe ((0L /: arr) (_ + _)) + (0L /: l)(_ + _) shouldBe ((0L /: arr)(_ + _)) l.foldRight(0L)(_ + _) shouldBe arr.foldRight(0L)(_ + _) - (l :\ 0L) (_ + _) shouldBe (arr :\ 0L) (_ + _) + (l :\ 0L)(_ + _) shouldBe (arr :\ 0L) (_ + _) } } @@ -490,7 +489,8 @@ trait LArrayBehaviour { } /** - * @author Taro L. Saito + * @author + * Taro L. Saito */ class LArrayFunctionTest extends LArraySpec with LArrayBehaviour { @@ -537,4 +537,3 @@ class LArrayFunctionTest extends LArraySpec with LArrayBehaviour { } } - diff --git a/larray/src/test/scala/xerial/larray/LArrayInputStreamTest.scala b/larray/src/test/scala/xerial/larray/LArrayInputStreamTest.scala index 668e14e..a5076d0 100644 --- a/larray/src/test/scala/xerial/larray/LArrayInputStreamTest.scala +++ b/larray/src/test/scala/xerial/larray/LArrayInputStreamTest.scala @@ -22,23 +22,24 @@ package xerial.larray +import wvlet.airspec.AirSpec import wvlet.log.io.IOUtil -class LArrayInputStreamTest extends LArraySpec { +class LArrayInputStreamTest extends AirSpec { - "LArrayInputStream" should { - "be created from LArray[A]" in { + test("LArrayInputStream") { + test("be created from LArray[A]") { val l = LArray(1, 3, 4, 5) debug(s"input ${l.mkString(", ")}") val in = LArrayInputStream(l) - IOUtil.readFully(in) {buf => + IOUtil.readFully(in) { buf => debug(s"buf length: ${buf.length}") val out = new LArrayOutputStream[Int] out.write(buf) val r = out.result debug(s"output ${r.mkString(", ")}") - l.sameElements(r) should be(true) + l.sameElements(r) shouldBe (true) } } } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/LArraySpec.scala b/larray/src/test/scala/xerial/larray/LArraySpec.scala index 1fae222..d06b581 100644 --- a/larray/src/test/scala/xerial/larray/LArraySpec.scala +++ b/larray/src/test/scala/xerial/larray/LArraySpec.scala @@ -22,31 +22,52 @@ package xerial.larray +import wvlet.airspec.AirSpec import org.scalatest._ -import java.io.{ByteArrayOutputStream, PrintStream} - +import org.scalatest.matchers.must.Matchers +import org.scalatest.wordspec.AnyWordSpec import wvlet.log.LogFormatter.SourceCodeLogFormatter import wvlet.log.{LogSupport, Logger} -import wvlet.log.io.ResourceReader + +import java.io.{ByteArrayOutputStream, PrintStream} import wvlet.log.io.Timer -import scala.language.implicitConversions +/** + * @author + * leo + */ +trait LArraySpec + extends AnyWordSpec + with Matchers + with Timer + with LogSupport + with BeforeAndAfterAll + with BeforeAndAfter + with BeforeAndAfterEach + with GivenWhenThen { + implicit def toTag(t: String) = Tag(t) -/** - * @author leo - */ -trait LArraySpec extends WordSpec with Matchers with ResourceReader with Timer with LogSupport with BeforeAndAfterAll with BeforeAndAfter with GivenWhenThen with BeforeAndAfterEach { + Logger.setDefaultFormatter(SourceCodeLogFormatter) - implicit def toTag(t:String) = Tag(t) + override protected def beforeAll(): Unit = { + // Run LogLevel scanner (log-test.properties or log.properties in classpath) every 1 minute + Logger.scheduleLogLevelScan + super.beforeAll() + } + + override protected def afterAll(): Unit = { + Logger.stopScheduledLogLevelScan + super.afterAll() + } /** - * Captures the output stream and returns the printed messages as a String - * @param body - * @tparam U - * @return - */ - def captureOut[U](body: => U) : String = { + * Captures the output stream and returns the printed messages as a String + * @param body + * @tparam U + * @return + */ + def captureOut[U](body: => U): String = { val out = new ByteArrayOutputStream Console.withOut(out) { body @@ -55,26 +76,25 @@ trait LArraySpec extends WordSpec with Matchers with ResourceReader with Timer w } /** - * Captures the output stream and returns the printed messages as a String - * @param body - * @tparam U - * @return - */ - def captureSystemOut[U](body: => U) : String = { + * Captures the output stream and returns the printed messages as a String + * @param body + * @tparam U + * @return + */ + def captureSystemOut[U](body: => U): String = { val prev = System.out - val b = new ByteArrayOutputStream - val out = new PrintStream(b) + val b = new ByteArrayOutputStream + val out = new PrintStream(b) try { System.setOut(out) body out.flush() - } - finally + } finally System.setOut(prev) new String(b.toByteArray) } - def captureErr[U](body: => U) : String = { + def captureErr[U](body: => U): String = { val out = new ByteArrayOutputStream Console.withErr(out) { body @@ -82,17 +102,4 @@ trait LArraySpec extends WordSpec with Matchers with ResourceReader with Timer w new String(out.toByteArray) } - Logger.setDefaultFormatter(SourceCodeLogFormatter) - - override protected def beforeAll(): Unit = { - // Run LogLevel scanner (log-test.properties or log.properties in classpath) every 1 minute - Logger.scheduleLogLevelScan - super.beforeAll() - } - - override protected def afterAll(): Unit = { - Logger.stopScheduledLogLevelScan - super.afterAll() - } - } diff --git a/larray/src/test/scala/xerial/larray/LArrayTest.scala b/larray/src/test/scala/xerial/larray/LArrayTest.scala index 87e3806..f4d2953 100644 --- a/larray/src/test/scala/xerial/larray/LArrayTest.scala +++ b/larray/src/test/scala/xerial/larray/LArrayTest.scala @@ -22,68 +22,65 @@ package xerial.larray +import wvlet.airspec.AirSpec +import wvlet.log.io.Timer + import scala.util.Random -import java.io.{FileInputStream, File, FileOutputStream} +import java.io.{File, FileInputStream, FileOutputStream} /** - * @author Taro L. Saito - */ -class LArrayTest extends LArraySpec { + * @author + * Taro L. Saito + */ +class LArrayTest extends AirSpec with Timer { val G: Long = 1024L * 1024L * 1024L - override def afterEach { - // MemoryAllocator.default.releaseAll - // System.gc() - } - - "LArray" should { - "have constructor" taggedAs("cc") in { + test("LArray") { + test("have constructor") { - val l = LArray(1, 2, 3) + val l = LArray(1, 2, 3) val l0 = LArray() val l1 = LArray(1) try { - l.size should be(3) - l(0) should be(1) - l(1) should be(2) - l(2) should be(3) + l.size shouldBe (3) + l(0) shouldBe (1) + l(1) shouldBe (2) + l(2) shouldBe (3) - l0.size should be(0) + l0.size shouldBe (0) try { l0.apply(3) fail("cannot reach here") - } - catch { + } catch { case e: Exception => // "OK" } - } - finally { + } finally { l.free l0.free l1.free } } - "have map/flatMap" taggedAs("map") in { - val l = LArray(1, 3, 5) + test("have map/flatMap") { + val l = LArray(1, 3, 5) def mul(v: Int) = v * 2 val m = l.map(mul).toArray - m.size should be(3) + m.size shouldBe (3) for (i <- 0L until l.size) - m(i.toInt) should be(mul(l(i))) + m(i.toInt) shouldBe (mul(l(i))) } - "read/write values correctly" in { + test("read/write values correctly") { info("read/write test") val step = 1L - val l = new LIntArray((0.1 * G).toLong) + val l = new LIntArray((0.1 * G).toLong) try { def v(i: Long) = (i * 2).toInt - for (i <- 0L until(l.size, step)) l(i) = v(i) + for (i <- 0L until (l.size, step)) l(i) = v(i) def loop(i: Long): Boolean = { if (i >= l.size) true @@ -91,14 +88,13 @@ class LArrayTest extends LArraySpec { l(i) == v(i) && loop(i + step) } - loop(0) should be(true) - } - finally { + loop(0) shouldBe (true) + } finally { l.free } } - "read/write data to Array[Byte]" taggedAs ("rw") in { + test("read/write data to Array[Byte]") { val l = LArray(1, 3) val b = new Array[Byte](l.byteLength.toInt) @@ -116,14 +112,13 @@ class LArrayTest extends LArraySpec { case l2: RawByteArray[_] => l2.readFromArray(b, 0, 0, b.length) debug(s"LArray2: [${l2.mkString(", ")}]") - l.sameElements(l2) should be(true) + l.sameElements(l2) shouldBe (true) case _ => fail("cannot reach here") } - } - "read/write data through ByteBuffer" taggedAs ("bb") in { + test("read/write data through ByteBuffer") { val l = LArray(1, 3, 5, 134, 34, -3, 2) debug(l.mkString(", ")) val bb = l.toDirectByteBuffer @@ -137,7 +132,7 @@ class LArrayTest extends LArraySpec { // read LArray from file { - val in = new FileInputStream(tmp).getChannel + val in = new FileInputStream(tmp).getChannel val fileSize = in.size() val newArray = new LByteArray(fileSize) @@ -148,49 +143,47 @@ class LArrayTest extends LArraySpec { val l2 = LArray.wrap[Int](newArray.byteLength, newArray.m) debug(l2.mkString(", ")) - l.sameElements(l2) should be(true) + l.sameElements(l2) shouldBe (true) in.close } // read LArray from File using LArrayBuilder { - val ib = LArray.newBuilder[Int] - val in = new FileInputStream(tmp).getChannel + val ib = LArray.newBuilder[Int] + val in = new FileInputStream(tmp).getChannel val fileSize = in.size() - var pos = 0L - while(pos < fileSize) { + var pos = 0L + while (pos < fileSize) { pos += in.transferTo(pos, fileSize - pos, ib) } val l2 = ib.result() debug(l2.mkString(", ")) - l.sameElements(l2) should be (true) + l.sameElements(l2) shouldBe (true) in.close } } - "provide initializer" taggedAs("init") in { - + test("provide initializer") { { val l = new LIntArray(1000) l(40) = 34 l.clear() - l.forall(_ == 0) should be (true) + l.forall(_ == 0) shouldBe (true) } - { val l = new LByteArray(1000) l(340) = 34.toByte l.clear() - l.forall(_ == 0) should be (true) + l.forall(_ == 0) shouldBe (true) } } - "compare its random access performance with native Scala array and its wrapper" in { - //val N = 1 * 1024 * 1024 * 1024 + test("compare its random access performance with native Scala array and its wrapper") { + // val N = 1 * 1024 * 1024 * 1024 val N = 1 * 1024 * 1024 info("benchmark has started..") val arr1 = new Array[Int](N) @@ -233,16 +226,15 @@ class LArrayTest extends LArraySpec { arr4(i) = 1 } } - } - finally { + } finally { arr2.free arr3.free } } - "compare sequential access performance" in { + test("compare sequential access performance") { - //val N = 1 * 1024 * 1024 * 1024 + // val N = 1 * 1024 * 1024 * 1024 val N = 64 * 1024 * 1024 info("benchmark has started..") val arr1 = new Array[Int](N) @@ -275,28 +267,25 @@ class LArrayTest extends LArraySpec { } } - } - finally { + } finally { arr2.free arr3.free } } - - "create large array" taggedAs ("la") in { + test("create large array") { info("large memory allocation test") for (i <- 0 until 10) { val arr = new LByteArray(2L * G) try { arr(arr.size - 1) = 134.toByte - } - finally { + } finally { arr.free } } } - "create view" taggedAs("view") in { + test("create view") { val l = LArray(1, 13, 4, 5) val v = l.view(1, 3) v.mkString(", ") shouldBe "13, 4" @@ -307,21 +296,19 @@ class LArrayTest extends LArraySpec { } + test("LByteArray") { - - "LByteArray" should { - - "have constructor" in { + test("have constructor") { val a = Array[Byte](1, 2, 3) val b = LArray[Byte](1, 5, 34) - b(0) should be(1.toByte) - b(1) should be(5.toByte) - b(2) should be(34.toByte) + b(0) shouldBe (1.toByte) + b(1) shouldBe (5.toByte) + b(2) shouldBe (34.toByte) } - "compare performance" taggedAs ("bp") in { + test("compare performance") { val N = (0.01 * G).toLong val a = new Array[Byte](N.toInt) @@ -368,6 +355,5 @@ class LArrayTest extends LArraySpec { b.free } - } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/LArrayTestWithPBT.scala b/larray/src/test/scala/xerial/larray/LArrayTestWithPBT.scala index 6aa1308..0f3ac2e 100644 --- a/larray/src/test/scala/xerial/larray/LArrayTestWithPBT.scala +++ b/larray/src/test/scala/xerial/larray/LArrayTestWithPBT.scala @@ -17,48 +17,42 @@ package xerial.larray import org.scalatest.prop.PropertyChecks import org.scalatest.prop.Checkers +import wvlet.airspec.spi.PropertyCheck /** - * Created with IntelliJ IDEA. - * User: hayato - * Date: 13/03/27 - * Time: 15:06 + * Created with IntelliJ IDEA. User: hayato Date: 13/03/27 Time: 15:06 */ -class LArrayTestWithPBT - extends LArraySpec with PropertyChecks with LArrayBehaviour { +class LArrayTestWithPBT extends LArraySpec with PropertyChecks with LArrayBehaviour { - implicit val config = PropertyCheckConfig(minSuccessful = 3, minSize = 1, maxSize = 10000) + implicit val config = + org.scalatest.prop.Configuration.PropertyCheckConfiguration(minSuccessful = 3, minSize = 1, sizeRange = 10000) - forAll { - (input: Array[Int]) => - s"int array [${input.take(10).mkString(", ")}, ...]" should { - behave like validArray(input) - behave like validIntArray(input) - } + forAll { (input: Array[Int]) => + s"int array [${input.take(10).mkString(", ")}, ...]" should { + behave like validArray(input) + behave like validIntArray(input) + } } - forAll { - (input: Array[Long]) => - s"long array [${input.take(10).mkString(", ")}, ...]" should { - behave like validArray(input) - behave like validLongArray(input) - } + forAll { (input: Array[Long]) => + s"long array [${input.take(10).mkString(", ")}, ...]" should { + behave like validArray(input) + behave like validLongArray(input) + } } - forAll { - (input: Array[Float]) => - s"float array [${input.take(10).mkString(", ")}, ...]" should { - behave like validArray(input) - behave like validFloatArray(input) - } + forAll { (input: Array[Float]) => + s"float array [${input.take(10).mkString(", ")}, ...]" should { + behave like validArray(input) + behave like validFloatArray(input) + } } - forAll { - (input: Array[Double]) => - s"double array [${input.take(10).mkString(", ")}, ...]" should { - behave like validArray(input) - behave like validDoubleArray(input) - } + forAll { (input: Array[Double]) => + s"double array [${input.take(10).mkString(", ")}, ...]" should { + behave like validArray(input) + behave like validDoubleArray(input) + } } "empty test" should { diff --git a/larray/src/test/scala/xerial/larray/LBitArrayTest.scala b/larray/src/test/scala/xerial/larray/LBitArrayTest.scala index c502f00..4c14ec0 100644 --- a/larray/src/test/scala/xerial/larray/LBitArrayTest.scala +++ b/larray/src/test/scala/xerial/larray/LBitArrayTest.scala @@ -22,61 +22,64 @@ package xerial.larray +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper + import scala.util.Random /** - * @author Taro L. Saito - */ + * @author + * Taro L. Saito + */ class LBitArrayTest extends LArraySpec with LArrayBehaviour { "LBitArray" should { "have constructor" in { val b = new LBitArray(6) - b.size should be (6) + b.size shouldBe (6) - b.clear + b.clear() b.on(1) - b.toString should be ("010000") + b.toString shouldBe ("010000") b.on(5) - b.toString should be ("010001") + b.toString shouldBe ("010001") } "set bits" in { val N = 100 val b = new LBitArray(N) - b.size should be (N) + b.size shouldBe (N) debug(b) - b.clear + b.clear() debug(b) - b.forall(_ == false) should be (true) - for(i <- 0L until b.length) { + b.forall(_ == false) shouldBe (true) + for (i <- 0L until b.length) { b.on(i) } - b.forall(_ == true) should be (true) + b.forall(_ == true) shouldBe (true) } "on and off specific bits" in { val b = new LBitArray(10000) b.fill - b.forall(_ == true) should be (true) + b.forall(_ == true) shouldBe (true) - for(pos <- Seq(91, 34, 5093, 443, 4)) { + for (pos <- Seq(91, 34, 5093, 443, 4)) { b.off(pos) - b(pos) should be (false) + b(pos) shouldBe (false) } } "have builder" in { - val b = LArray.newBuilder[Boolean] + val b = LArray.newBuilder[Boolean] val in = Seq(true, false, false, true, true, false, false, true, true) - in.foreach( b += _ ) + in.foreach(b += _) val l = b.result() debug(l) - l.toString should be (in.map(v => if(v) "1" else "0").mkString) + l.toString shouldBe (in.map(v => if (v) "1" else "0").mkString) } @@ -86,9 +89,9 @@ class LBitArrayTest extends LArraySpec with LArrayBehaviour { } "behave like valid LArray for large input" should { - val input2 = (for(i <- 0 until 150) yield { Random.nextBoolean }).toArray.toSeq + val input2 = (for (i <- 0 until 150) yield { Random.nextBoolean() }).toArray.toSeq behave like validArray(input2) } } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/MappedLByteArrayTest.scala b/larray/src/test/scala/xerial/larray/MappedLByteArrayTest.scala index c90eee5..0025871 100644 --- a/larray/src/test/scala/xerial/larray/MappedLByteArrayTest.scala +++ b/larray/src/test/scala/xerial/larray/MappedLByteArrayTest.scala @@ -22,67 +22,70 @@ package xerial.larray +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper + import java.io.File /** - * @author Taro L. Saito + * @author + * Taro L. Saito */ class MappedLByteArrayTest extends LArraySpec { - "MappedLByteArray" should { - - "create memory mapped file" taggedAs ("simple") in { - - import xerial.larray._ - - val f = File.createTempFile("mmap", ".larray", new File("target")) - f.deleteOnExit() - - val L = 100L - val m = new MappedLByteArray(f, 0, L) - m.size shouldBe L - for (i <- 0 Until m.size) { - m(i) = i.toByte - } - - trace(m.mkString(", ")) - val mc = m.slice(0) - m.close() - - val m2 = LArray.loadFrom[Byte](f) - mc.sameElements(m2) should be(true) - - val mOffset = new MappedLByteArray(f, 3, L - 3) - trace(mOffset.mkString(", ")) - - mc.slice(3).sameElements(mOffset) should be(true) - - //mOffset.flush - //m.free - //m2.free - } - - "create large memory mapped file more than 2GB" taggedAs ("large") in { - - val f = File.createTempFile("mmap", ".larray", new File("target")) - f.deleteOnExit() - - val G = 1024L * 1024 * 1024 - val m = new MappedLByteArray(f, 0, 2L * G + 1024) - val offset = 100 - - val v = 34.toByte - m(2L * G + offset) = v - m.close() - - f.length() shouldBe m.size - - val view = new MappedLByteArray(f, 2L * G, 1024) - view(offset) shouldBe v - view.close() - - f.delete() - } - - } -} \ No newline at end of file +// "MappedLByteArray" should { +// +// "create memory mapped file" taggedAs ("simple") in { +// +// import xerial.larray._ +// +// val f = File.createTempFile("mmap", ".larray", new File("target")) +// f.deleteOnExit() +// +// val L = 100L +// val m = new MappedLByteArray(f, 0, L) +// m.size shouldBe L +// for (i <- 0 Until m.size) { +// m(i) = i.toByte +// } +// +// trace(m.mkString(", ")) +// val mc = m.slice(0) +// m.close() +// +// val m2 = LArray.loadFrom[Byte](f) +// mc.sameElements(m2) shouldBe (true) +// +// val mOffset = new MappedLByteArray(f, 3, L - 3) +// trace(mOffset.mkString(", ")) +// +// mc.slice(3).sameElements(mOffset) shouldBe (true) +// +// // mOffset.flush +// // m.free +// // m2.free +// } +// +// "create large memory mapped file more than 2GB" taggedAs ("large") in { +// +// val f = File.createTempFile("mmap", ".larray", new File("target")) +// f.deleteOnExit() +// +// val G = 1024L * 1024 * 1024 +// val m = new MappedLByteArray(f, 0, 2L * G + 1024) +// val offset = 100 +// +// val v = 34.toByte +// m(2L * G + offset) = v +// m.close() +// +// f.length() shouldBe m.size +// +// val view = new MappedLByteArray(f, 2L * G, 1024) +// view(offset) shouldBe v +// view.close() +// +// f.delete() +// } +// +// } +} diff --git a/larray/src/test/scala/xerial/larray/SnappyCompressTest.scala b/larray/src/test/scala/xerial/larray/SnappyCompressTest.scala index 482f692..2be48fc 100644 --- a/larray/src/test/scala/xerial/larray/SnappyCompressTest.scala +++ b/larray/src/test/scala/xerial/larray/SnappyCompressTest.scala @@ -22,39 +22,41 @@ package xerial.larray +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper + import java.io.File -import xerial.larray.mmap.MMapMode +//import xerial.larray.mmap.MMapMode import org.xerial.snappy.Snappy import xerial.larray.buffer.LBuffer /** - * @author Taro L. Saito - */ + * @author + * Taro L. Saito + */ class SnappyCompressTest extends LArraySpec { - implicit class AsRaw[A](l:LArray[A]) { + implicit class AsRaw[A](l: LArray[A]) { def address = l.asInstanceOf[RawByteArray[Int]].address } - "Snappy" should { + "Snappy" must { import xerial.larray._ - "support zero-copy compress with LBuffer" taggedAs("zerocopy") in { - val N = 10000 + "support zero-copy compress with LBuffer" taggedAs ("zerocopy") in { + val N = 10000 val buf = new LBuffer(4 * N) - for(i <- 0 Until N) { + for (i <- 0 Until N) { buf.putFloat(i * 4, math.sin(i * 0.01).toFloat) } - val arr = buf.toArray + val arr = buf.toArray val bufSize = buf.size.toInt - val maxCompressedLength = Snappy.maxCompressedLength(bufSize) val GN = 10 - val R = 5 + val R = 5 info("Start compression benchmark") time("compress", repeat = GN, blockRepeat = R) { @@ -74,8 +76,8 @@ class SnappyCompressTest extends LArraySpec { } } - val compressedLBuffer = new LBuffer(maxCompressedLength) - val compressedArray = new Array[Byte](maxCompressedLength) + val compressedLBuffer = new LBuffer(maxCompressedLength) + val compressedArray = new Array[Byte](maxCompressedLength) val compressedArrayDain = new Array[Byte](maxCompressedLength) Snappy.rawCompress(buf.address, bufSize, compressedLBuffer.address) @@ -104,67 +106,66 @@ class SnappyCompressTest extends LArraySpec { } - "compress LArray" in { - val l = (for (i <- 0 until 3000) yield math.toDegrees(math.sin(i/360)).toInt).toLArray - val maxLen = Snappy.maxCompressedLength(l.byteLength.toInt) + val l = (for (i <- 0 until 3000) yield math.toDegrees(math.sin(i / 360)).toInt).toLArray + val maxLen = Snappy.maxCompressedLength(l.byteLength.toInt) val compressedBuf = LArray.of[Byte](maxLen) val compressedLen = Snappy.rawCompress(l.address, l.byteLength, compressedBuf.address) - val compressed = compressedBuf.slice(0, compressedLen) + val compressed = compressedBuf.slice(0, compressedLen) val uncompressedLength = Snappy.uncompressedLength(compressed.address, compressed.byteLength) - val uncompressed = LArray.of[Int](uncompressedLength / 4) + val uncompressed = LArray.of[Int](uncompressedLength / 4) Snappy.rawUncompress(compressed.address, compressed.byteLength, uncompressed.address) debug(s"byteLength:${l.byteLength}, max compressed length:$maxLen ,compressed length:$compressedLen") - l.sameElements(uncompressed) should be (true) + l.sameElements(uncompressed) shouldBe (true) } - "compress LIntArray" taggedAs("it") in { - val N = 100000000L - val l = new LIntArray(N) - info(f"preparing data set. N=$N%,d") - for(i <- 0 Until N) l(i) = math.toDegrees(math.sin(i/360)).toInt - - debug("compressing the data") - val maxLen = Snappy.maxCompressedLength(l.byteLength.toInt) - val compressedBuf = LArray.of[Byte](maxLen) - val compressedLen = Snappy.rawCompress(l.address, l.byteLength, compressedBuf.address) - val compressed = compressedBuf.slice(0, compressedLen) - val f = File.createTempFile("snappy", ".dat", new File("target")) - f.deleteOnExit() - compressed.saveTo(f) - - debug("decompressing the data") - val b = LArray.mmap(f, MMapMode.READ_ONLY) - val len = Snappy.uncompressedLength(b.address, b.length) - val decompressed = new LIntArray(len / 4) - Snappy.rawUncompress(b.address, b.length, decompressed.address) - b.close - - debug(f"l.length:${l.length}%,d, decompressed.length:${decompressed.length}%,d") - - l.sameElements(decompressed) should be (true) - info("start bench") - time("iterate", repeat=10) { - block("new array") { - var sum = 0L - var i = 0 - while(i < l.length) { - sum += l(i) - i += 1 - } - } - - block("decompressed") { - var sum = 0L - var i = 0 - while(i < decompressed.length) { - sum += decompressed(i) - i += 1 - } - } - } - } +// "compress LIntArray" taggedAs ("it") in { +// val N = 100000000L +// val l = new LIntArray(N) +// info(f"preparing data set. N=$N%,d") +// for (i <- 0 Until N) l(i) = math.toDegrees(math.sin(i / 360)).toInt +// +// debug("compressing the data") +// val maxLen = Snappy.maxCompressedLength(l.byteLength.toInt) +// val compressedBuf = LArray.of[Byte](maxLen) +// val compressedLen = Snappy.rawCompress(l.address, l.byteLength, compressedBuf.address) +// val compressed = compressedBuf.slice(0, compressedLen) +// val f = File.createTempFile("snappy", ".dat", new File("target")) +// f.deleteOnExit() +// compressed.saveTo(f) +// +// debug("decompressing the data") +// val b = LArray.mmap(f, MMapMode.READ_ONLY) +// val len = Snappy.uncompressedLength(b.address, b.length) +// val decompressed = new LIntArray(len / 4) +// Snappy.rawUncompress(b.address, b.length, decompressed.address) +// b.close +// +// debug(f"l.length:${l.length}%,d, decompressed.length:${decompressed.length}%,d") +// +// l.sameElements(decompressed) shouldBe (true) +// info("start bench") +// time("iterate", repeat = 10) { +// block("new array") { +// var sum = 0L +// var i = 0 +// while (i < l.length) { +// sum += l(i) +// i += 1 +// } +// } +// +// block("decompressed") { +// var sum = 0L +// var i = 0 +// while (i < decompressed.length) { +// sum += decompressed(i) +// i += 1 +// } +// } +// } +// } } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/UInt32ArrayTest.scala b/larray/src/test/scala/xerial/larray/UInt32ArrayTest.scala index 73fe1c8..c6bbd18 100644 --- a/larray/src/test/scala/xerial/larray/UInt32ArrayTest.scala +++ b/larray/src/test/scala/xerial/larray/UInt32ArrayTest.scala @@ -22,42 +22,45 @@ package xerial.larray +import wvlet.airspec.AirSpec + /** - * @author Taro L. Saito - */ -class UInt32ArrayTest extends LArraySpec { + * @author + * Taro L. Saito + */ +class UInt32ArrayTest extends AirSpec { - "UInt32Array" should { + test("UInt32Array") { - "record values larger than 2G" in { + test("record values larger than 2G") { - val u = new UInt32Array(10) - val v : Long = Int.MaxValue.toLong + 10L + val u = new UInt32Array(10) + val v: Long = Int.MaxValue.toLong + 10L u(0) = v - u(0) should be (v) + u(0) shouldBe (v) u(1) = 3141341 - u(1) should be (3141341L) + u(1) shouldBe (3141341L) val v2 = Integer.MAX_VALUE.toLong * 2L u(2) = v2 - u(2) should be (v2) + u(2) shouldBe (v2) } - "have builder" in { + test("have builder") { val b = UInt32Array.newBuilder b += 1 b += Int.MaxValue.toLong + 3L b += 4 - val u = b.result + val u = b.result() - u.size should be(3) - u(0) should be (1L) - u(1) should be (Int.MaxValue.toLong + 3L) - u(2) should be (4L) + u.size shouldBe (3) + u(0) shouldBe (1L) + u(1) shouldBe (Int.MaxValue.toLong + 3L) + u(2) shouldBe (4L) } } -} \ No newline at end of file +} diff --git a/larray/src/test/scala/xerial/larray/example/LArrayExampleTest.scala b/larray/src/test/scala/xerial/larray/example/LArrayExampleTest.scala index c74cea5..f5c35ad 100644 --- a/larray/src/test/scala/xerial/larray/example/LArrayExampleTest.scala +++ b/larray/src/test/scala/xerial/larray/example/LArrayExampleTest.scala @@ -1,45 +1,45 @@ -/*-------------------------------------------------------------------------- - * Copyright 2013 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// -// LArrayExampleTest.scala -// Since: 2013/03/26 15:04 -// -//-------------------------------------- - -package xerial.larray.example - -import xerial.larray.LArraySpec - - -/** - * @author Taro L. Saito - */ -class LArrayExampleTest extends LArraySpec { - - "LArrayExample" should { - "run Scala API" in { - val out = captureOut(new LArrayExample) - out should (include ("done.")) - } - - "run Java API" in { - val out = captureSystemOut(LArrayJavaExample.main(Array.empty[String])) - out should (include ("done.")) - } - - } -} \ No newline at end of file +/*-------------------------------------------------------------------------- + * Copyright 2013 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// +// LArrayExampleTest.scala +// Since: 2013/03/26 15:04 +// +//-------------------------------------- + +package xerial.larray.example + +import xerial.larray.LArraySpec + +/** + * @author + * Taro L. Saito + */ +class LArrayExampleTest extends LArraySpec { + + "LArrayExample" should { + "run Scala API" in { + val out = captureOut(new LArrayExample) + out must include("done.") + } + + "run Java API" in { + val out = captureSystemOut(LArrayJavaExample.main(Array.empty[String])) + out must include("done.") + } + + } +} diff --git a/project/build.properties b/project/build.properties index 27e88aa..8b9a0b0 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.13 +sbt.version=1.8.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index 6fd82b8..6d3015b 100755 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,10 @@ -addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.7.9") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") -addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M15") -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.3") -addSbtPlugin("com.typesafe.sbt" % "sbt-multi-jvm" % "0.3.11") -addSbtPlugin("com.eed3si9n" % "sbt-doge" % "0.1.5") +// sbt-scoverage upgraded to scala-xml 2.1.0, but other sbt-plugins and Scala compilier 2.12 uses scala-xml 1.x.x +ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % "always" + +addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.17") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.15") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.0") +addSbtPlugin("com.typesafe.sbt" % "sbt-multi-jvm" % "0.4.0") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") + +scalacOptions ++= Seq("-deprecation", "-feature") diff --git a/sbt b/sbt index bd47e3b..b1cdf71 100755 --- a/sbt +++ b/sbt @@ -2,32 +2,61 @@ # # A more capable sbt runner, coincidentally also called sbt. # Author: Paul Phillips +# https://github.com/paulp/sbt-extras +# +# Generated from http://www.opensource.org/licenses/bsd-license.php +# Copyright (c) 2011, Paul Phillips. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -o pipefail -declare -r sbt_release_version="0.13.13" -declare -r sbt_unreleased_version="0.13.13" +declare -r sbt_release_version="1.8.0" +declare -r sbt_unreleased_version="1.8.0" -declare -r latest_212="2.12.0" -declare -r latest_211="2.11.8" -declare -r latest_210="2.10.6" +declare -r latest_213="2.13.10" +declare -r latest_212="2.12.17" +declare -r latest_211="2.11.12" +declare -r latest_210="2.10.7" declare -r latest_29="2.9.3" declare -r latest_28="2.8.2" declare -r buildProps="project/build.properties" -declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" +declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases" declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" -declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases" -declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots" +declare -r sbt_launch_mvn_release_repo="https://repo1.maven.org/maven2" +declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots" -declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m" -declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" +declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18" +declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy -Dsbt.coursier.home=project/.coursier" declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new declare sbt_explicit_version declare verbose noshare batch trace_level -declare sbt_saved_stty debugUs declare java_cmd="java" declare sbt_launch_dir="$HOME/.sbt/launchers" @@ -39,31 +68,40 @@ declare -a java_args scalac_args sbt_commands residual_args # args to jvm/sbt via files or environment variables declare -a extra_jvm_opts extra_sbt_opts -echoerr () { echo >&2 "$@"; } -vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } -die () { echo "Aborting: $@" ; exit 1; } - -# restore stty settings (echo in particular) -onSbtRunnerExit() { - [[ -n "$sbt_saved_stty" ]] || return - vlog "" - vlog "restoring stty: $sbt_saved_stty" - stty "$sbt_saved_stty" - unset sbt_saved_stty +echoerr() { echo >&2 "$@"; } +vlog() { [[ -n "$verbose" ]] && echoerr "$@"; } +die() { + echo "Aborting: $*" + exit 1 } -# save stty and trap exit, to ensure echo is re-enabled if we are interrupted. -trap onSbtRunnerExit EXIT -sbt_saved_stty="$(stty -g 2>/dev/null)" -vlog "Saved stty: $sbt_saved_stty" +setTrapExit() { + # save stty and trap exit, to ensure echo is re-enabled if we are interrupted. + SBT_STTY="$(stty -g 2>/dev/null)" + export SBT_STTY + + # restore stty settings (echo in particular) + onSbtRunnerExit() { + [ -t 0 ] || return + vlog "" + vlog "restoring stty: $SBT_STTY" + stty "$SBT_STTY" + } + + vlog "saving stty: $SBT_STTY" + trap onSbtRunnerExit EXIT +} # this seems to cover the bases on OSX, and someone will # have to tell me about the others. -get_script_path () { +get_script_path() { local path="$1" - [[ -L "$path" ]] || { echo "$path" ; return; } + [[ -L "$path" ]] || { + echo "$path" + return + } - local target="$(readlink "$path")" + local -r target="$(readlink "$path")" if [[ "${target:0:1}" == "/" ]]; then echo "$target" else @@ -71,10 +109,12 @@ get_script_path () { fi } -declare -r script_path="$(get_script_path "$BASH_SOURCE")" -declare -r script_name="${script_path##*/}" +script_path="$(get_script_path "${BASH_SOURCE[0]}")" +declare -r script_path +script_name="${script_path##*/}" +declare -r script_name -init_default_option_file () { +init_default_option_file() { local overriding_var="${!1}" local default_file="$2" if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then @@ -86,82 +126,82 @@ init_default_option_file () { echo "$default_file" } -declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" -declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" +sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" +sbtx_opts_file="$(init_default_option_file SBTX_OPTS .sbtxopts)" +jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" -build_props_sbt () { - [[ -r "$buildProps" ]] && \ +build_props_sbt() { + [[ -r "$buildProps" ]] && grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' } -update_build_props_sbt () { - local ver="$1" - local old="$(build_props_sbt)" - - [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && { - perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps" - grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps" - - vlog "!!!" - vlog "!!! Updated file $buildProps setting sbt.version to: $ver" - vlog "!!! Previous value was: $old" - vlog "!!!" - } -} - -set_sbt_version () { +set_sbt_version() { sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version export sbt_version } -url_base () { +url_base() { local version="$1" case "$version" in - 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; - 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;; + 0.7.*) echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;; + 0.10.*) echo "$sbt_launch_ivy_release_repo" ;; 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;; 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_ivy_snapshot_repo" ;; - 0.*) echo "$sbt_launch_ivy_release_repo" ;; - *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_mvn_snapshot_repo" ;; - *) echo "$sbt_launch_mvn_release_repo" ;; + echo "$sbt_launch_ivy_snapshot_repo" ;; + 0.*) echo "$sbt_launch_ivy_release_repo" ;; + *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss" + echo "$sbt_launch_mvn_snapshot_repo" ;; + *) echo "$sbt_launch_mvn_release_repo" ;; esac } -make_url () { +make_url() { local version="$1" local base="${sbt_launch_repo:-$(url_base "$version")}" case "$version" in - 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;; - 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.7.*) echo "$base/sbt-launch-0.7.7.jar" ;; + 0.10.*) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; - 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; - *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;; esac } -addJava () { vlog "[addJava] arg = '$1'" ; java_args+=("$1"); } -addSbt () { vlog "[addSbt] arg = '$1'" ; sbt_commands+=("$1"); } -addScalac () { vlog "[addScalac] arg = '$1'" ; scalac_args+=("$1"); } -addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); } +addJava() { + vlog "[addJava] arg = '$1'" + java_args+=("$1") +} +addSbt() { + vlog "[addSbt] arg = '$1'" + sbt_commands+=("$1") +} +addScalac() { + vlog "[addScalac] arg = '$1'" + scalac_args+=("$1") +} +addResidual() { + vlog "[residual] arg = '$1'" + residual_args+=("$1") +} + +addResolver() { addSbt "set resolvers += $1"; } + +addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } -addResolver () { addSbt "set resolvers += $1"; } -addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } -setThisBuild () { - vlog "[addBuild] args = '$@'" +setThisBuild() { + vlog "[addBuild] args = '$*'" local key="$1" && shift - addSbt "set $key in ThisBuild := $@" + addSbt "set $key in ThisBuild := $*" } -setScalaVersion () { +setScalaVersion() { [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' addSbt "++ $1" } -setJavaHome () { +setJavaHome() { java_cmd="$1/bin/java" setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))" export JAVA_HOME="$1" @@ -169,13 +209,26 @@ setJavaHome () { export PATH="$JAVA_HOME/bin:$PATH" } -getJavaVersion() { "$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \"; } +getJavaVersion() { + local -r str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"') + + # java -version on java8 says 1.8.x + # but on 9 and 10 it's 9.x.y and 10.x.y. + if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then + echo "${BASH_REMATCH[1]}" + # Fixes https://github.com/dwijnand/sbt-extras/issues/326 + elif [[ "$str" =~ ^([0-9]+)(\..*)?(-ea)?$ ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ -n "$str" ]]; then + echoerr "Can't parse java version from: $str" + fi +} checkJava() { # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME - [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" - [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" + [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" + [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" if [[ -n "$java" ]]; then pathJavaVersion=$(getJavaVersion java) @@ -189,31 +242,34 @@ checkJava() { fi } -java_version () { - local version=$(getJavaVersion "$java_cmd") +java_version() { + local -r version=$(getJavaVersion "$java_cmd") vlog "Detected Java version: $version" - echo "${version:2:1}" + echo "$version" } +is_apple_silicon() { [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; } + # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+ -default_jvm_opts () { - local v="$(java_version)" - if [[ $v -ge 8 ]]; then +default_jvm_opts() { + local -r v="$(java_version)" + if [[ $v -ge 17 ]]; then + echo "$default_jvm_opts_common" + elif [[ $v -ge 10 ]]; then + if is_apple_silicon; then + # As of Dec 2020, JVM for Apple Silicon (M1) doesn't support JVMCI + echo "$default_jvm_opts_common" + else + echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler" + fi + elif [[ $v -ge 8 ]]; then echo "$default_jvm_opts_common" else echo "-XX:MaxPermSize=384m $default_jvm_opts_common" fi } -build_props_scala () { - if [[ -r "$buildProps" ]]; then - versionLine="$(grep '^build.scala.versions' "$buildProps")" - versionString="${versionLine##build.scala.versions=}" - echo "${versionString%% .*}" - fi -} - -execRunner () { +execRunner() { # print the arguments one to a line, quoting any containing spaces vlog "# Executing command line:" && { for arg; do @@ -228,38 +284,39 @@ execRunner () { vlog "" } - [[ -n "$batch" ]] && exec /dev/null; then + if command -v curl >/dev/null 2>&1; then curl --fail --silent --location "$url" --output "$jar" - elif which wget >/dev/null; then + elif command -v wget >/dev/null 2>&1; then wget -q -O "$jar" "$url" fi } && [[ -r "$jar" ]] } -acquire_sbt_jar () { +acquire_sbt_jar() { { sbt_jar="$(jar_file "$sbt_version")" [[ -r "$sbt_jar" ]] @@ -268,11 +325,66 @@ acquire_sbt_jar () { [[ -r "$sbt_jar" ]] } || { sbt_jar="$(jar_file "$sbt_version")" - download_url "$(make_url "$sbt_version")" "$sbt_jar" + jar_url="$(make_url "$sbt_version")" + + echoerr "Downloading sbt launcher for ${sbt_version}:" + echoerr " From ${jar_url}" + echoerr " To ${sbt_jar}" + + download_url "${jar_url}" "${sbt_jar}" + + case "${sbt_version}" in + 0.*) + vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check" + echo "" + ;; + *) verify_sbt_jar "${sbt_jar}" ;; + esac } } -usage () { +verify_sbt_jar() { + local jar="${1}" + local md5="${jar}.md5" + md5url="$(make_url "${sbt_version}").md5" + + echoerr "Downloading sbt launcher ${sbt_version} md5 hash:" + echoerr " From ${md5url}" + echoerr " To ${md5}" + + download_url "${md5url}" "${md5}" >/dev/null 2>&1 + + if command -v md5sum >/dev/null 2>&1; then + if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v md5 >/dev/null 2>&1; then + if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v openssl >/dev/null 2>&1; then + if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + else + echoerr "Could not find an MD5 command" + return 1 + fi +} + +usage() { set_sbt_version cat < Run the specified file as a scala script # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) - -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version - -sbt-version use the specified version of sbt (default: $sbt_release_version) - -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version - -sbt-jar use the specified jar as the sbt launcher - -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) - -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) + -sbt-version use the specified version of sbt (default: $sbt_release_version) + -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version + -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version + -sbt-jar use the specified jar as the sbt launcher + -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) + -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) # scala version (default: as chosen by sbt) - -28 use $latest_28 - -29 use $latest_29 - -210 use $latest_210 - -211 use $latest_211 - -212 use $latest_212 - -scala-home use the scala build at the specified directory - -scala-version use the specified version of scala - -binary-version use the specified scala version when searching for dependencies + -28 use $latest_28 + -29 use $latest_29 + -210 use $latest_210 + -211 use $latest_211 + -212 use $latest_212 + -213 use $latest_213 + -scala-home use the scala build at the specified directory + -scala-version use the specified version of scala + -binary-version use the specified scala version when searching for dependencies # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) - -java-home alternate JAVA_HOME + -java-home alternate JAVA_HOME # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found - $(default_jvm_opts) - JVM_OPTS environment variable holding either the jvm args directly, or - the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') - Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. - -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) - -Dkey=val pass -Dkey=val directly to the jvm - -J-X pass option -X directly to the jvm (-J is stripped) + $(default_jvm_opts) + JVM_OPTS environment variable holding either the jvm args directly, or + the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') + Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. + -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) + -Dkey=val pass -Dkey=val directly to the jvm + -J-X pass option -X directly to the jvm (-J is stripped) # passing options to sbt, OR to this runner - SBT_OPTS environment variable holding either the sbt args directly, or - the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') - Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. - -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) - -S-X add -X to sbt's scalacOptions (-S is stripped) + SBT_OPTS environment variable holding either the sbt args directly, or + the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') + Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. + -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) + -S-X add -X to sbt's scalacOptions (-S is stripped) + + # passing options exclusively to this runner + SBTX_OPTS environment variable holding either the sbt-extras args directly, or + the reference to a file containing sbt-extras args if given path is prepended by '@' (e.g. '@/etc/sbtxopts') + Note: "@"-file is overridden by local '.sbtxopts' or '-sbtx-opts' argument. + -sbtx-opts file containing sbt-extras args (if not given, .sbtxopts in project root is used if present) EOM + exit 0 } -process_args () { - require_arg () { +process_args() { + require_arg() { local type="$1" local opt="$2" local arg="$3" @@ -358,49 +472,56 @@ process_args () { } while [[ $# -gt 0 ]]; do case "$1" in - -h|-help) usage; exit 1 ;; - -v) verbose=true && shift ;; - -d) addSbt "--debug" && shift ;; - -w) addSbt "--warn" && shift ;; - -q) addSbt "--error" && shift ;; - -x) debugUs=true && shift ;; - -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; - -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; - -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; - -no-share) noshare=true && shift ;; - -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; - -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; - -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; - -offline) addSbt "set offline := true" && shift ;; - -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; - -batch) batch=true && shift ;; - -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; - -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; - - -sbt-create) sbt_create=true && shift ;; - -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -h | -help) usage ;; + -v) verbose=true && shift ;; + -d) addSbt "--debug" && shift ;; + -w) addSbt "--warn" && shift ;; + -q) addSbt "--error" && shift ;; + -x) shift ;; # currently unused + -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; + -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; + + -no-colors) addJava "-Dsbt.log.noformat=true" && addJava "-Dsbt.color=false" && shift ;; + -sbt-create) sbt_create=true && shift ;; + -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; + -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; + -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; + -no-share) noshare=true && shift ;; + -offline) addSbt "set offline in Global := true" && shift ;; + -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; + -batch) batch=true && shift ;; + -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; + -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; + -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; - -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; - -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; - -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; - -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; - -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; - -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; - -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; - -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; - -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; - -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; - - -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; - -S*) addScalac "${1:2}" && shift ;; - -28) setScalaVersion "$latest_28" && shift ;; - -29) setScalaVersion "$latest_29" && shift ;; - -210) setScalaVersion "$latest_210" && shift ;; - -211) setScalaVersion "$latest_211" && shift ;; - -212) setScalaVersion "$latest_212" && shift ;; - new) sbt_new=true && sbt_explicit_version="$sbt_release_version" && addResidual "$1" && shift ;; - *) addResidual "$1" && shift ;; + -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; + -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; + -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; + -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; + + -28) setScalaVersion "$latest_28" && shift ;; + -29) setScalaVersion "$latest_29" && shift ;; + -210) setScalaVersion "$latest_210" && shift ;; + -211) setScalaVersion "$latest_211" && shift ;; + -212) setScalaVersion "$latest_212" && shift ;; + -213) setScalaVersion "$latest_213" && shift ;; + + -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; + -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; + -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; + -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; + -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; + -sbtx-opts) require_arg path "$1" "$2" && sbtx_opts_file="$2" && shift 2 ;; + -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; + + -D*) addJava "$1" && shift ;; + -J*) addJava "${1:2}" && shift ;; + -S*) addScalac "${1:2}" && shift ;; + + new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; + + *) addResidual "$1" && shift ;; esac done } @@ -412,19 +533,31 @@ process_args "$@" readConfigFile() { local end=false until $end; do - read || end=true + read -r || end=true [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY" - done < "$1" + done <"$1" } # if there are file/environment sbt_opts, process again so we # can supply args to this runner if [[ -r "$sbt_opts_file" ]]; then vlog "Using sbt options defined in file $sbt_opts_file" - while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") + while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then vlog "Using sbt options defined in variable \$SBT_OPTS" - extra_sbt_opts=( $SBT_OPTS ) + IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS" +else + vlog "No extra sbt options have been defined" +fi + +# if there are file/environment sbtx_opts, process again so we +# can supply args to this runner +if [[ -r "$sbtx_opts_file" ]]; then + vlog "Using sbt options defined in file $sbtx_opts_file" + while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbtx_opts_file") +elif [[ -n "$SBTX_OPTS" && ! ("$SBTX_OPTS" =~ ^@.*) ]]; then + vlog "Using sbt options defined in variable \$SBTX_OPTS" + IFS=" " read -r -a extra_sbt_opts <<<"$SBTX_OPTS" else vlog "No extra sbt options have been defined" fi @@ -443,25 +576,24 @@ checkJava # only exists in 0.12+ setTraceLevel() { case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; - *) setThisBuild traceLevel $trace_level ;; + "0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;; + *) setThisBuild traceLevel "$trace_level" ;; esac } # set scalacOptions if we were given any -S opts -[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\"" +[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[*]}\"" -# Update build.properties on disk to set explicit version - sbt gives us no choice -[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && update_build_props_sbt "$sbt_explicit_version" +[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && addJava "-Dsbt.version=$sbt_explicit_version" vlog "Detected sbt version $sbt_version" if [[ -n "$sbt_script" ]]; then - residual_args=( $sbt_script ${residual_args[@]} ) + residual_args=("$sbt_script" "${residual_args[@]}") else # no args - alert them there's stuff in here - (( argumentCount > 0 )) || { + ((argumentCount > 0)) || { vlog "Starting $script_name: invoke with -help for other options" - residual_args=( shell ) + residual_args=(shell) } fi @@ -477,6 +609,7 @@ EOM } # pick up completion if present; todo +# shellcheck disable=SC1091 [[ -r .sbt_completion.sh ]] && source .sbt_completion.sh # directory to store sbt launchers @@ -486,7 +619,7 @@ EOM # no jar? download it. [[ -r "$sbt_jar" ]] || acquire_sbt_jar || { # still no jar? uh-oh. - echo "Download failed. Obtain the jar manually and place it at $sbt_jar" + echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar" exit 1 } @@ -496,12 +629,12 @@ if [[ -n "$noshare" ]]; then done else case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* | "0.12."* ) + "0.7."* | "0.10."* | "0.11."* | "0.12."*) [[ -n "$sbt_dir" ]] || { sbt_dir="$HOME/.sbt/$sbt_version" vlog "Using $sbt_dir as sbt dir, -sbt-dir to override." } - ;; + ;; esac if [[ -n "$sbt_dir" ]]; then @@ -511,58 +644,21 @@ fi if [[ -r "$jvm_opts_file" ]]; then vlog "Using jvm options defined in file $jvm_opts_file" - while read opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") + while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then vlog "Using jvm options defined in \$JVM_OPTS variable" - extra_jvm_opts=( $JVM_OPTS ) + IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS" else vlog "Using default jvm options" - extra_jvm_opts=( $(default_jvm_opts) ) + IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)" fi # traceLevel is 0.12+ [[ -n "$trace_level" ]] && setTraceLevel -main () { - execRunner "$java_cmd" \ - "${extra_jvm_opts[@]}" \ - "${java_args[@]}" \ - -jar "$sbt_jar" \ - "${sbt_commands[@]}" \ - "${residual_args[@]}" -} - -# sbt inserts this string on certain lines when formatting is enabled: -# val OverwriteLine = "\r\u001BM\u001B[2K" -# ...in order not to spam the console with a million "Resolving" lines. -# Unfortunately that makes it that much harder to work with when -# we're not going to print those lines anyway. We strip that bit of -# line noise, but leave the other codes to preserve color. -mainFiltered () { - local ansiOverwrite='\r\x1BM\x1B[2K' - local excludeRegex=$(egrep -v '^#|^$' ~/.sbtignore | paste -sd'|' -) - - echoLine () { - local line="$1" - local line1="$(echo "$line" | sed 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code. - local line2="$(echo "$line1" | sed 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this. - - if [[ $line2 =~ $excludeRegex ]]; then - [[ -n $debugUs ]] && echo "[X] $line1" - else - [[ -n $debugUs ]] && echo " $line1" || echo "$line1" - fi - } - - echoLine "Starting sbt with output filtering enabled." - main | while read -r line; do echoLine "$line"; done -} - -# Only filter if there's a filter file and we don't see a known interactive command. -# Obviously this is super ad hoc but I don't know how to improve on it. Testing whether -# stdin is a terminal is useless because most of my use cases for this filtering are -# exactly when I'm at a terminal, running sbt non-interactively. -shouldFilter () { [[ -f ~/.sbtignore ]] && ! egrep -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } - -# run sbt -if shouldFilter; then mainFiltered; else main; fi +execRunner "$java_cmd" \ + "${extra_jvm_opts[@]}" \ + "${java_args[@]}" \ + -jar "$sbt_jar" \ + "${sbt_commands[@]}" \ + "${residual_args[@]}"