MR-JAR rewrite of MMapDirectory with JDK-19 preview Panama APIs (>= JDK-19-ea+23) (#912)

This uses Gradle's auto-provisioning to compile Java 19 classes and build a multi-release JAR from them. Please make sure to regenerate gradle.properties (delete it) or change "org.gradle.java.installations.auto-download" to "true"

Author: uschindler

gradle/generation/local-settings.gradle

@@ -98,12 +98,12 @@ org.gradle.workers.max=${maxWorkers}
 # Maximum number of test JVMs forked per test task.
 tests.jvms=${testsJvms}
 
-# Disable auto JVM provisioning (we don't use toolchains yet but want no surprises).
-org.gradle.java.installations.auto-download=false
+# Enable auto JVM provisioning.
+org.gradle.java.installations.auto-download=true
 
 # Set these to enable automatic JVM location discovery.
-org.gradle.java.installations.fromEnv=JDK11,JDK12,JDK13,JDK14,JDK15,JDK16,JDK17
-org.gradle.java.installations.paths=(custom paths)
+org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME
+#org.gradle.java.installations.paths=(custom paths)
 
 """, "UTF-8")
 

gradle/java/javac.gradle

@@ -79,3 +79,48 @@ allprojects {
     }
   }
 }
+
+configure(project(":lucene:core")) {
+  plugins.withType(JavaPlugin) {
+    sourceSets {
+      main19 {
+        java {
+          srcDirs = ['src/java19']
+        }
+      }
+    }
+
+    configurations {
+      // Inherit any dependencies from the main source set.
+      main19Implementation.extendsFrom implementation
+    }
+    
+    dependencies {
+      // We need the main classes to compile our Java 19 pieces.
+      main19Implementation sourceSets.main.output
+    }
+
+    tasks.named('compileMain19Java').configure {
+      javaCompiler = javaToolchains.compilerFor {
+        languageVersion = JavaLanguageVersion.of(19)
+      }
+      
+      // undo alternative JDK support:
+      options.forkOptions.javaHome = null
+      
+      sourceCompatibility = 19
+      targetCompatibility = 19
+      options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
+    }
+    
+    tasks.named('jar').configure {
+      into('META-INF/versions/19') {
+        from sourceSets.main19.output
+      }
+
+      manifest.attributes(
+        'Multi-Release': 'true'
+      )
+    }
+  }
+}

gradle/testing/defaults-tests.gradle

@@ -124,6 +124,10 @@ allprojects {
       // (if the runner JVM does not support them, it will fail tests):
       jvmArgs '--add-modules', 'jdk.unsupported,jdk.management'
 
+      if (rootProject.runtimeJavaVersion == JavaVersion.VERSION_19) {
+        jvmArgs '--enable-preview'
+      }
+
       systemProperty 'java.util.logging.config.file', file("${resources}/logging.properties")
       systemProperty 'java.awt.headless', 'true'
       systemProperty 'jdk.map.althashing.threshold', '0'

gradle/validation/ecj-lint.gradle

@@ -34,7 +34,9 @@ allprojects {
     // Create a [sourceSetName]EcjLint task for each source set
     // with a non-empty java.srcDirs. These tasks are then
     // attached to project's "ecjLint" task.
-    def lintTasks = sourceSets.collect { sourceSet ->
+
+    // TODO: Better way to disable on our MR-JAR
+    def lintTasks = sourceSets.findAll { it.name != 'main19' }.collect { sourceSet ->
       def srcDirs = sourceSet.java.sourceDirectories
           .filter { dir -> dir.exists() }
 

gradle/validation/error-prone.gradle

@@ -58,6 +58,12 @@ allprojects { prj ->
       }
 
       tasks.withType(JavaCompile) { task ->
+        // TODO: Better way to disable on our MR-JAR
+        if (task.name == 'compileMain19Java') {
+          options.errorprone.enabled = false
+          return
+        }
+
         task.dependsOn ":checkJdkInternalsExportedToGradle"
 
         options.errorprone.disableWarningsInGeneratedCode = true

gradle/validation/forbidden-apis.gradle

@@ -115,6 +115,11 @@ allprojects { prj ->
       inputs.dir(file(resources))
     }
 
+    // TODO: Remove this when forbiddenapis supports Java 19 class files
+    tasks.matching { it.name == "forbiddenApisMain19" }.all {
+      enabled = false
+    }
+
     // We rely on resolved configurations to compute the relevant set of rule
     // files for forbiddenApis. Since we don't want to resolve these configurations until
     // the task is executed, we can't really use them as task inputs properly. This is a

lucene/CHANGES.txt

@@ -141,6 +141,15 @@ New Features
   SortField#getOptimizeSortWithIndexedData were introduce to provide
   an option to disable sort optimization for various sort fields. (Mayya Sharipova)
 
+* GITHUB#912: Support for Java 19 foreign memory support was added. Applications started
+  with command line parameter "java --enable-preview" will automatically use the new
+  foreign memory API of Java 19 to access indexes on disk with MMapDirectory. This is
+  an opt-in feature and requires explicit Java command line flag! When enabled, Lucene logs
+  a notice using java.util.logging. Please test thoroughly and report bugs/slowness to Lucene's
+  mailing list. When the new API is used, MMapDirectory will mmap Lucene indexes in chunks of
+  16 GiB (instead of 1 GiB) and indexes closed while queries are running can no longer crash
+  the JVM.  (Uwe Schindler)
+
 Improvements
 ---------------------
 * LUCENE-10592: Build HNSW Graph on indexing. (Mayya Sharipova, Adrien Grand, Julie Tibshirani)