From 320282a9920fb90b1fcf93da3da5879271265da9 Mon Sep 17 00:00:00 2001 From: MBWhite Date: Wed, 17 Dec 2025 10:29:06 +0000 Subject: [PATCH 1/8] fix: improve the javadoc generation Make all components put javadoc into a consistent root. This achieves the following - means the javadoc for the generated protobuf can be kept quiet - as we can't adjust the javadoc here - there is a single place for javadoc that could be published - the isthmus and core and protobuf projects can be interlinked - isthmus can be linked with the calcite javadoc - overview pages can be added - all under a versioned directory for easy maintence To date there isn't (AFAIK) a stightforward maintainable way to merge javadoc form multi-component projects Signed-off-by: MBWhite --- core/build.gradle.kts | 64 +++++++++++++++++++ .../main/java/io/substrait/package-info.java | 4 ++ .../examples/util/SubstraitStringify.java | 1 - isthmus/build.gradle.kts | 26 ++++++++ overview.html | 22 +++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 overview.html diff --git a/core/build.gradle.kts b/core/build.gradle.kts index df0f3f364..e6c3a42a2 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -273,3 +273,67 @@ protobuf { generateProtoTasks { all().configureEach { dependsOn(submodulesUpdate) } } protoc { artifact = "com.google.protobuf:protoc:" + libs.protoc.get().getVersion() } } + +val protoJavaDir = layout.buildDirectory.dir("generated/sources/proto/main/java") + +// First pass: Javadoc for generated protobufs — ignore warnings. +tasks.register("javadocProto") { + group = JavaBasePlugin.DOCUMENTATION_GROUP + description = "Generate Javadoc for protobuf-generated sources (warnings suppressed)." + + // Only the generated proto sources + setSource(fileTree(protoJavaDir) { include("**/*.java") }) + + // Use the main source set classpath to resolve types referenced by the generated code + classpath = sourceSets["main"].compileClasspath + + // Destination separate from main Javadoc + destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/core-proto").get().asFile + + // Make sure protobufs are generated before Javadoc runs + dependsOn("generateProto") + + // Suppress warnings/doclint for protobuf pass + (options as StandardJavadocDocletOptions).apply { + // Disable doclint entirely + addBooleanOption("Xdoclint:none", true) + // Be quiet + addBooleanOption("quiet", true) + // Encoding is good practice + encoding = "UTF-8" + } + + // Do not fail the build if javadoc finds issues in generated sources + isFailOnError = false +} + +// Second pass: Javadoc for main code, excluding the generated protobuf sources. +tasks.named("javadoc") { + mustRunAfter("javadocProto") + description = "Generate Javadoc for main sources (excludes protobuf-generated sources)." + + // Exclude the protobuf-generated directory from the main pass + val protoDirFile = protoJavaDir.get().asFile + exclude { spec -> spec.file.toPath().startsWith(protoDirFile.toPath()) } + + // Keep normal behavior for main javadoc (warnings allowed to show/fail if you want) + (options as StandardJavadocDocletOptions).apply { + encoding = "UTF-8" + destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/core").get().asFile + + addStringOption("overview", "${rootProject.projectDir}/overview.html") + addStringOption("link", "../core-proto") + } +} + +// Bundle both passes into the Javadoc JAR used for publishing. +tasks.named("javadocJar") { + val shared = rootProject.layout.buildDirectory.dir("docs/${version}").get().asFile + if (!shared.exists()) { + println("Creating a dir for javadoc ${rootProject.buildDir}/docs/${version}") + shared.mkdirs() + } + + // Ensure both javadoc tasks have produced outputs + dependsOn(tasks.named("javadocProto"), tasks.named("javadoc")) +} diff --git a/core/src/main/java/io/substrait/package-info.java b/core/src/main/java/io/substrait/package-info.java index a62caaf60..e90c4c332 100644 --- a/core/src/main/java/io/substrait/package-info.java +++ b/core/src/main/java/io/substrait/package-info.java @@ -1,2 +1,6 @@ +/** + * The {@code io.substrait} package provides core classes and interfaces for working with the + * Substrait specification, which defines a portable representation of query plans and expressions. + */ @org.immutables.value.Value.Style(allowedClasspathAnnotations = {java.lang.Override.class}) package io.substrait; diff --git a/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java b/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java index 36a991a6c..18638d8af 100644 --- a/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java +++ b/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java @@ -73,7 +73,6 @@ public SubstraitStringify() { /** * Explains the Sustrait plan * - * @param plan Subsrait plan * @return List of strings; typically these would then be logged or sent to stdout */ public static List explain(io.substrait.plan.Plan plan) { diff --git a/isthmus/build.gradle.kts b/isthmus/build.gradle.kts index ad6ce6230..ca36c2c6a 100644 --- a/isthmus/build.gradle.kts +++ b/isthmus/build.gradle.kts @@ -147,3 +147,29 @@ tasks { sourceSets { test { proto.srcDirs("src/test/resources/extensions") } } protobuf { protoc { artifact = "com.google.protobuf:protoc:" + libs.protoc.get().getVersion() } } + +tasks.named("javadoc") { + description = "Generate Javadoc for main sources (excludes protobuf-generated sources)." + + // Keep normal behavior for main javadoc (warnings allowed to show/fail if you want) + (options as StandardJavadocDocletOptions).apply { + encoding = "UTF-8" + destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/isthmus").get().asFile + + addStringOption("link", "../core-proto") + addStringOption("link", "../core") + addStringOption("link", "https://calcite.apache.org/javadocAggregate/") + } +} + +// Bundle both passes into the Javadoc JAR used for publishing. +tasks.named("javadocJar") { + val shared = rootProject.layout.buildDirectory.dir("docs/${version}").get().asFile + if (!shared.exists()) { + println("Creating a dir for javadoc ${rootProject.buildDir}/${version}") + shared.mkdirs() + } + + // Ensure javadoc tasks have produced output + dependsOn(tasks.named("javadoc")) +} diff --git a/overview.html b/overview.html new file mode 100644 index 000000000..1bb905f8e --- /dev/null +++ b/overview.html @@ -0,0 +1,22 @@ + +
+

Core Substrait Java API

+

+ The {@code io.substrait} package provides core classes and interfaces for + working with the Substrait specification, which defines a portable + representation of query plans and expressions. +

+ +

Components

+
    +
  • core Core classes and interfaces
    • +
  • + core-protobuf The ProtoBuf wrapper + classes +
    • +
  • + Isthmus API Utility module that uses + Calcite to convert SQL to/from Substrait +
    • +
+ From fbb6e49d5d710f50ffbb26beec16c30ee12a5847 Mon Sep 17 00:00:00 2001 From: MBWhite Date: Mon, 5 Jan 2026 16:31:10 +0000 Subject: [PATCH 2/8] fix: review comments Signed-off-by: MBWhite --- .../sessions/kotlin-compiler-743613223953325844.salive | 0 .../io/substrait/examples/util/SubstraitStringify.java | 8 +++++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 build-logic/.kotlin/sessions/kotlin-compiler-743613223953325844.salive diff --git a/build-logic/.kotlin/sessions/kotlin-compiler-743613223953325844.salive b/build-logic/.kotlin/sessions/kotlin-compiler-743613223953325844.salive new file mode 100644 index 000000000..e69de29bb diff --git a/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java b/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java index 18638d8af..76c515117 100644 --- a/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java +++ b/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java @@ -71,8 +71,10 @@ public SubstraitStringify() { } /** - * Explains the Sustrait plan + * Explains the Substrait plan * + * @param plan Substrait Plan + * * @return List of strings; typically these would then be logged or sent to stdout */ public static List explain(io.substrait.plan.Plan plan) { @@ -92,9 +94,9 @@ public static List explain(io.substrait.plan.Plan plan) { } /** - * Explains the Sustrait relation + * Explains the Substrait relation * - * @param rel Subsrait relation + * @param rel Substrait relation * @return List of strings; typically these would then be logged or sent to stdout */ public static List explain(io.substrait.relation.Rel rel) { From 653dab18f8bd25e514a1f13a0f5a9c00daf4ca3e Mon Sep 17 00:00:00 2001 From: MBWhite Date: Mon, 5 Jan 2026 16:31:47 +0000 Subject: [PATCH 3/8] fix: review comments Signed-off-by: MBWhite --- .../.kotlin/sessions/kotlin-compiler-743613223953325844.salive | 0 .../main/java/io/substrait/examples/util/SubstraitStringify.java | 1 - 2 files changed, 1 deletion(-) delete mode 100644 build-logic/.kotlin/sessions/kotlin-compiler-743613223953325844.salive diff --git a/build-logic/.kotlin/sessions/kotlin-compiler-743613223953325844.salive b/build-logic/.kotlin/sessions/kotlin-compiler-743613223953325844.salive deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java b/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java index 76c515117..fbbb57c1c 100644 --- a/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java +++ b/examples/substrait-spark/src/main/java/io/substrait/examples/util/SubstraitStringify.java @@ -74,7 +74,6 @@ public SubstraitStringify() { * Explains the Substrait plan * * @param plan Substrait Plan - * * @return List of strings; typically these would then be logged or sent to stdout */ public static List explain(io.substrait.plan.Plan plan) { From 8ad5684889c57295ed1c03c906d67405e734c405 Mon Sep 17 00:00:00 2001 From: MBWhite Date: Wed, 7 Jan 2026 14:36:50 +0000 Subject: [PATCH 4/8] fix: review comments Signed-off-by: MBWhite --- overview.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/overview.html b/overview.html index 1bb905f8e..b090898d5 100644 --- a/overview.html +++ b/overview.html @@ -11,11 +11,11 @@

Components

From 0c563fb6a81fa291fe3f567761d44876db0f397b Mon Sep 17 00:00:00 2001 From: MBWhite Date: Fri, 9 Jan 2026 09:46:41 +0000 Subject: [PATCH 5/8] fix: review comments Signed-off-by: MBWhite --- core/build.gradle.kts | 2 +- overview.html => core/src/main/javadoc/overview.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename overview.html => core/src/main/javadoc/overview.html (88%) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index e6c3a42a2..8c9d549a5 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -321,7 +321,7 @@ tasks.named("javadoc") { encoding = "UTF-8" destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/core").get().asFile - addStringOption("overview", "${rootProject.projectDir}/overview.html") + addStringOption("overview", "${rootProject.projectDir}/core/src/main/javadoc/overview.html") addStringOption("link", "../core-proto") } } diff --git a/overview.html b/core/src/main/javadoc/overview.html similarity index 88% rename from overview.html rename to core/src/main/javadoc/overview.html index b090898d5..c75444241 100644 --- a/overview.html +++ b/core/src/main/javadoc/overview.html @@ -9,7 +9,7 @@

Core Substrait Java API

Components

    -
  • core Core classes and interfaces
    • +
  • core - Core classes and interfaces
  • core-protobuf - The Proto Buffers wrapper classes From 1a9f817e5982024091c0264cd7796ddc3f6620a4 Mon Sep 17 00:00:00 2001 From: MBWhite Date: Wed, 14 Jan 2026 11:30:06 +0000 Subject: [PATCH 6/8] fix: resync with main Signed-off-by: MBWhite --- substrait | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrait b/substrait index 92d2e757a..3c25b1b3e 160000 --- a/substrait +++ b/substrait @@ -1 +1 @@ -Subproject commit 92d2e757a330f9c973bb566817dc92afd1badcb2 +Subproject commit 3c25b1b3eaadecba6d10af6b3dd0fe038d0c5993 From c82414e7b04ee5f2ee606e059191346b77254079 Mon Sep 17 00:00:00 2001 From: MBWhite Date: Fri, 16 Jan 2026 11:56:36 +0000 Subject: [PATCH 7/8] fix: submodule fixes again Signed-off-by: MBWhite --- substrait | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrait b/substrait index 3c25b1b3e..92d2e757a 160000 --- a/substrait +++ b/substrait @@ -1 +1 @@ -Subproject commit 3c25b1b3eaadecba6d10af6b3dd0fe038d0c5993 +Subproject commit 92d2e757a330f9c973bb566817dc92afd1badcb2 From 776b5d88a261963a84c4103de92e0bf4690f5277 Mon Sep 17 00:00:00 2001 From: MBWhite Date: Fri, 23 Jan 2026 14:00:55 +0000 Subject: [PATCH 8/8] fix: add in immutable classes and linking fixes Signed-off-by: MBWhite --- core/build.gradle.kts | 52 +++++++++++++++++++++-- core/src/main/javadoc/overview-proto.html | 24 +++++++++++ core/src/main/javadoc/overview.html | 7 +++ isthmus/build.gradle.kts | 10 +++-- isthmus/src/main/javadoc/overview.html | 21 +++++++++ 5 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 core/src/main/javadoc/overview-proto.html create mode 100644 isthmus/src/main/javadoc/overview.html diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 8c9d549a5..6dd40e142 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -276,13 +276,18 @@ protobuf { val protoJavaDir = layout.buildDirectory.dir("generated/sources/proto/main/java") +val immuteableJavaDir = layout.buildDirectory.dir("generated/sources/annotationProcessor/java/main") + // First pass: Javadoc for generated protobufs — ignore warnings. tasks.register("javadocProto") { + dependsOn("generateProto", "compileJava") + group = JavaBasePlugin.DOCUMENTATION_GROUP description = "Generate Javadoc for protobuf-generated sources (warnings suppressed)." // Only the generated proto sources setSource(fileTree(protoJavaDir) { include("**/*.java") }) + // source(fileTree(immuteableJavaDir) { include("**/*.java") }) // Use the main source set classpath to resolve types referenced by the generated code classpath = sourceSets["main"].compileClasspath @@ -301,6 +306,46 @@ tasks.register("javadocProto") { addBooleanOption("quiet", true) // Encoding is good practice encoding = "UTF-8" + addStringOption( + "overview", + "${rootProject.projectDir}/core/src/main/javadoc/overview-proto.html", + ) + } + + // Do not fail the build if javadoc finds issues in generated sources + isFailOnError = false +} + +tasks.register("javadocImmutable") { + dependsOn("compileJava", "javadoc") + + group = JavaBasePlugin.DOCUMENTATION_GROUP + description = "Generate Javadoc for immutable-generated sources (warnings suppressed)." + + // Only the generated proto sources + setSource(fileTree(immuteableJavaDir) { include("**/*.java") }) + // source(fileTree(immuteableJavaDir) { include("**/*.java") }) + + // Use the main source set classpath to resolve types referenced by the generated code + classpath = sourceSets["main"].compileClasspath + + // Destination separate from main Javadoc + destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/immutable").get().asFile + + // Suppress warnings/doclint for protobuf pass + (options as StandardJavadocDocletOptions).apply { + // Disable doclint entirely + addBooleanOption("Xdoclint:none", true) + // Be quiet + addBooleanOption("quiet", true) + // Encoding is good practice + encoding = "UTF-8" + addStringOption( + "overview", + "${rootProject.projectDir}/core/src/main/javadoc/overview-immutable.html", + ) + links("../core/") + links("../core-proto/") } // Do not fail the build if javadoc finds issues in generated sources @@ -309,12 +354,13 @@ tasks.register("javadocProto") { // Second pass: Javadoc for main code, excluding the generated protobuf sources. tasks.named("javadoc") { - mustRunAfter("javadocProto") + dependsOn("javadocProto") description = "Generate Javadoc for main sources (excludes protobuf-generated sources)." // Exclude the protobuf-generated directory from the main pass val protoDirFile = protoJavaDir.get().asFile exclude { spec -> spec.file.toPath().startsWith(protoDirFile.toPath()) } + source(fileTree(immuteableJavaDir) { include("**/*.java") }) // Keep normal behavior for main javadoc (warnings allowed to show/fail if you want) (options as StandardJavadocDocletOptions).apply { @@ -322,7 +368,7 @@ tasks.named("javadoc") { destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/core").get().asFile addStringOption("overview", "${rootProject.projectDir}/core/src/main/javadoc/overview.html") - addStringOption("link", "../core-proto") + links("../core-proto/") } } @@ -335,5 +381,5 @@ tasks.named("javadocJar") { } // Ensure both javadoc tasks have produced outputs - dependsOn(tasks.named("javadocProto"), tasks.named("javadoc")) + dependsOn(tasks.named("javadocProto"), tasks.named("javadoc"), tasks.named("javadocImmutable")) } diff --git a/core/src/main/javadoc/overview-proto.html b/core/src/main/javadoc/overview-proto.html new file mode 100644 index 000000000..a28a18284 --- /dev/null +++ b/core/src/main/javadoc/overview-proto.html @@ -0,0 +1,24 @@ + +
    +

    Core Substrait Protocol Buffers

    +

    + This package contains the Protocol Buffer generated classes for the Substrait + specification. These classes provide the serialization format for Substrait + query plans and expressions, enabling cross-language and cross-platform + interoperability. +

    + +

    Components

    +
      +
    • + core - Core Substrait classes and interfaces +
      • +
    • + core-protobuf - The Proto Buffers wrapper classes +
      • +
    • + Isthmus API - Utility module that uses + Calcite to convert SQL to/from Substrait +
      • +
    + diff --git a/core/src/main/javadoc/overview.html b/core/src/main/javadoc/overview.html index c75444241..bbd92817f 100644 --- a/core/src/main/javadoc/overview.html +++ b/core/src/main/javadoc/overview.html @@ -7,6 +7,13 @@

    Core Substrait Java API

    representation of query plans and expressions.

    +

    + Also includes immutable value objects generated by the Immutables + annotation processor. These classes provide type-safe, immutable data structures + for working with Substrait concepts, ensuring thread safety and preventing + accidental mutations. +

    +

    Components

    • core - Core classes and interfaces
      • diff --git a/isthmus/build.gradle.kts b/isthmus/build.gradle.kts index ca36c2c6a..8b0618b3e 100644 --- a/isthmus/build.gradle.kts +++ b/isthmus/build.gradle.kts @@ -149,16 +149,18 @@ sourceSets { test { proto.srcDirs("src/test/resources/extensions") } } protobuf { protoc { artifact = "com.google.protobuf:protoc:" + libs.protoc.get().getVersion() } } tasks.named("javadoc") { - description = "Generate Javadoc for main sources (excludes protobuf-generated sources)." + dependsOn(":core:javadoc", ":core:javadocProto") + description = "Generate Javadoc for main sources." // Keep normal behavior for main javadoc (warnings allowed to show/fail if you want) (options as StandardJavadocDocletOptions).apply { encoding = "UTF-8" destinationDir = rootProject.layout.buildDirectory.dir("docs/${version}/isthmus").get().asFile - addStringOption("link", "../core-proto") - addStringOption("link", "../core") - addStringOption("link", "https://calcite.apache.org/javadocAggregate/") + addStringOption("overview", "${rootProject.projectDir}/isthmus/src/main/javadoc/overview.html") + links("../core-proto/") + links("../core/") + links("https://calcite.apache.org/javadocAggregate/") } } diff --git a/isthmus/src/main/javadoc/overview.html b/isthmus/src/main/javadoc/overview.html new file mode 100644 index 000000000..36cd67d54 --- /dev/null +++ b/isthmus/src/main/javadoc/overview.html @@ -0,0 +1,21 @@ + +
      +

      Isthmus - SQL to Substrait Converter

      +

      + The {@code io.substrait.isthmus} package provides utilities for converting + between SQL and Substrait using Apache Calcite. Isthmus enables bidirectional + translation of SQL queries to Substrait plans and vice versa. +

      + +

      Components

      +
        +
      • + core - Core Substrait classes and interfaces +
        • +
      • + core-protobuf - The Proto Buffers wrapper + classes +
        • +
      • Isthmus API - SQL to/from Substrait conversion
        • +
      +