Updates to changelog processing after docs redesign (elastic#89463)

# Conflicts:
#	build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java
#	build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc
#	docs/changelog/83345.yaml

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java

@@ -74,6 +74,7 @@ static String generateMigrationFile(QualifiedVersion version, String template, L
         bindings.put("deprecationsByNotabilityByArea", deprecationsByNotabilityByArea);
         bindings.put("isElasticsearchSnapshot", version.isSnapshot());
         bindings.put("majorDotMinor", version.major() + "." + version.minor());
+        bindings.put("majorDotMinorDotRevision", version.major() + "." + version.minor() + "." + version.revision());
         bindings.put("majorMinor", String.valueOf(version.major()) + version.minor());
         bindings.put("nextMajor", (version.major() + 1) + ".0");
         bindings.put("version", version);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java

@@ -99,6 +99,7 @@ public Highlight getHighlight() {
 
     public void setHighlight(Highlight highlight) {
         this.highlight = highlight;
+        if (this.highlight != null) this.highlight.pr = this.pr;
     }
 
     public Breaking getBreaking() {
@@ -160,6 +161,7 @@ public static class Highlight {
         private boolean notable;
         private String title;
         private String body;
+        private Integer pr;
 
         public boolean isNotable() {
             return notable;
@@ -189,6 +191,10 @@ public String getAnchor() {
             return generatedAnchor(this.title);
         }
 
+        public Integer getPr() {
+            return pr;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java

@@ -37,6 +37,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
@@ -55,11 +56,13 @@ public class GenerateReleaseNotesTask extends DefaultTask {
     private final RegularFileProperty releaseNotesTemplate;
     private final RegularFileProperty releaseHighlightsTemplate;
     private final RegularFileProperty breakingChangesTemplate;
+    private final RegularFileProperty migrationIndexTemplate;
 
     private final RegularFileProperty releaseNotesIndexFile;
     private final RegularFileProperty releaseNotesFile;
     private final RegularFileProperty releaseHighlightsFile;
     private final RegularFileProperty breakingChangesMigrationFile;
+    private final RegularFileProperty migrationIndexFile;
 
     private final GitWrapper gitWrapper;
 
@@ -71,11 +74,13 @@ public GenerateReleaseNotesTask(ObjectFactory objectFactory, ExecOperations exec
         releaseNotesTemplate = objectFactory.fileProperty();
         releaseHighlightsTemplate = objectFactory.fileProperty();
         breakingChangesTemplate = objectFactory.fileProperty();
+        migrationIndexTemplate = objectFactory.fileProperty();
 
         releaseNotesIndexFile = objectFactory.fileProperty();
         releaseNotesFile = objectFactory.fileProperty();
         releaseHighlightsFile = objectFactory.fileProperty();
         breakingChangesMigrationFile = objectFactory.fileProperty();
+        migrationIndexFile = objectFactory.fileProperty();
 
         gitWrapper = new GitWrapper(execOperations);
     }
@@ -136,6 +141,13 @@ public void executeTask() throws IOException {
             this.breakingChangesMigrationFile.get().getAsFile(),
             entries
         );
+
+        LOGGER.info("Updating migration/index...");
+        MigrationIndexGenerator.update(
+            getMinorVersions(versions),
+            this.migrationIndexTemplate.get().getAsFile(),
+            this.migrationIndexFile.get().getAsFile()
+        );
     }
 
     /**
@@ -146,11 +158,19 @@ public void executeTask() throws IOException {
      */
     @VisibleForTesting
     static Set<QualifiedVersion> getVersions(GitWrapper gitWrapper, String currentVersion) {
-        QualifiedVersion v = QualifiedVersion.of(currentVersion);
-        final String pattern = "v" + v.major() + ".*";
-        Set<QualifiedVersion> versions = gitWrapper.listVersions(pattern).collect(toSet());
-        versions.add(v);
-        return versions;
+        QualifiedVersion qualifiedVersion = QualifiedVersion.of(currentVersion);
+        final String pattern = "v" + qualifiedVersion.major() + ".*";
+        // We may be generating notes for a minor version prior to the latest minor, so we need to filter out versions that are too new.
+        return Stream.concat(gitWrapper.listVersions(pattern).filter(v -> v.isBefore(qualifiedVersion)), Stream.of(qualifiedVersion))
+            .collect(toSet());
+    }
+
+    /**
+     * Convert set of QualifiedVersion to MinorVersion by deleting all but the major and minor components.
+     */
+    @VisibleForTesting
+    static Set<MinorVersion> getMinorVersions(Set<QualifiedVersion> versions) {
+        return versions.stream().map(MinorVersion::of).collect(toSet());
     }
 
     /**
@@ -319,6 +339,15 @@ public void setBreakingChangesTemplate(RegularFile file) {
         this.breakingChangesTemplate.set(file);
     }
 
+    @InputFile
+    public RegularFileProperty getMigrationIndexTemplate() {
+        return migrationIndexTemplate;
+    }
+
+    public void setMigrationIndexTemplate(RegularFile file) {
+        this.migrationIndexTemplate.set(file);
+    }
+
     @OutputFile
     public RegularFileProperty getReleaseNotesIndexFile() {
         return releaseNotesIndexFile;
@@ -354,4 +383,13 @@ public RegularFileProperty getBreakingChangesMigrationFile() {
     public void setBreakingChangesMigrationFile(RegularFile file) {
         this.breakingChangesMigrationFile.set(file);
     }
+
+    @OutputFile
+    public RegularFileProperty getMigrationIndexFile() {
+        return migrationIndexFile;
+    }
+
+    public void setMigrationIndexFile(RegularFile file) {
+        this.migrationIndexFile.set(file);
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.release;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import static java.util.Comparator.reverseOrder;
+
+/**
+ * This class ensures that the migrate/index page has the appropriate anchors and include directives
+ * for the current repository version.
+ */
+public class MigrationIndexGenerator {
+
+    static void update(Set<MinorVersion> versions, File indexTemplate, File indexFile) throws IOException {
+        try (FileWriter indexFileWriter = new FileWriter(indexFile)) {
+            indexFileWriter.write(generateFile(versions, Files.readString(indexTemplate.toPath())));
+        }
+    }
+
+    @VisibleForTesting
+    static String generateFile(Set<MinorVersion> versionsSet, String template) throws IOException {
+        final Set<MinorVersion> versions = new TreeSet<>(reverseOrder());
+        versions.addAll(versionsSet);
+        final List<String> includeVersions = versions.stream().map(MinorVersion::underscore).collect(Collectors.toList());
+
+        final Map<String, Object> bindings = new HashMap<>();
+        bindings.put("versions", versions);
+        bindings.put("includeVersions", includeVersions);
+
+        return TemplateUtils.render(template, bindings);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MinorVersion.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.release;
+
+import java.util.Comparator;
+import java.util.Objects;
+
+/**
+ * Encapsulates comparison and printing logic for an x.y.
+ */
+public record MinorVersion(int major, int minor) implements Comparable<MinorVersion> {
+    /**
+     * Converts a QualifiedVersion into a MinorVersion by deleting all but the major and minor components.
+     */
+    public static MinorVersion of(final QualifiedVersion v) {
+        Objects.requireNonNull(v);
+        return new MinorVersion(v.major(), v.minor());
+    }
+
+    @Override
+    public String toString() {
+        return major + "." + minor;
+    }
+
+    /** Generate version string with underscore instead of dot */
+    public String underscore() {
+        return major + "_" + minor;
+    }
+
+    private static final Comparator<MinorVersion> COMPARATOR = Comparator.comparing((MinorVersion v) -> v.major)
+        .thenComparing(v -> v.minor);
+
+    @Override
+    public int compareTo(MinorVersion other) {
+        return COMPARATOR.compare(this, other);
+    }
+
+    public boolean isBefore(MinorVersion other) {
+        return this.compareTo(other) < 0;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java

@@ -17,6 +17,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,21 +44,14 @@ static String generateFile(QualifiedVersion version, String template, List<Chang
             final int major = version.major();
             for (int minor = version.minor() - 1; minor >= 0; minor--) {
                 String majorMinor = major + "." + minor;
-                priorVersions.add(
-                    "{ref-bare}/"
-                        + majorMinor
-                        + "/release-highlights"
-                        + (minor <= 6 ? "-" + majorMinor + ".0" : "")
-                        + ".html["
-                        + majorMinor
-                        + "]"
-                );
+                priorVersions.add("{ref-bare}/" + majorMinor + "/release-highlights.html[" + majorMinor + "]");
             }
         }
 
         final Map<Boolean, List<ChangelogEntry.Highlight>> groupedHighlights = entries.stream()
             .map(ChangelogEntry::getHighlight)
             .filter(Objects::nonNull)
+            .sorted(Comparator.comparingInt(ChangelogEntry.Highlight::getPr))
             .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList()));
 
         final List<ChangelogEntry.Highlight> notableHighlights = groupedHighlights.getOrDefault(true, List.of());

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java

@@ -68,9 +68,6 @@ static String generateFile(String template, QualifiedVersion version, Set<Change
 
     private static Map<String, Map<String, List<ChangelogEntry>>> buildChangelogBreakdown(Set<ChangelogEntry> changelogs) {
         Map<String, Map<String, List<ChangelogEntry>>> changelogsByTypeByArea = changelogs.stream()
-            // Special case - we have a changelog file that isn't in the 'known-issue' or 'security' areas, but
-            // doesn't have an ES PR for it.
-            .filter(each -> each.getPr() == null || each.getPr() != -1)
             .collect(
                 groupingBy(
                     // Entries with breaking info are always put in the breaking section

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java

@@ -29,13 +29,6 @@
  */
 public class ReleaseNotesIndexGenerator {
 
-    // Some versions where never released or were pulled. They shouldn't be listed.
-    private static Set<QualifiedVersion> EXCLUDED_VERSIONS = Set.of(
-        QualifiedVersion.of("7.0.1"),
-        QualifiedVersion.of("7.13.3"),
-        QualifiedVersion.of("7.13.4")
-    );
-
     static void update(Set<QualifiedVersion> versions, File indexTemplate, File indexFile) throws IOException {
         try (FileWriter indexFileWriter = new FileWriter(indexFile)) {
             indexFileWriter.write(generateFile(versions, Files.readString(indexTemplate.toPath())));
@@ -47,19 +40,9 @@ static String generateFile(Set<QualifiedVersion> versionsSet, String template) t
         final Set<QualifiedVersion> versions = new TreeSet<>(reverseOrder());
 
         // For the purpose of generating the index, snapshot versions are the same as released versions. Prerelease versions are not.
-        versionsSet.stream()
-            .filter(v -> EXCLUDED_VERSIONS.contains(v) == false)
-            .map(v -> v.isSnapshot() ? v.withoutQualifier() : v)
-            .forEach(versions::add);
+        versionsSet.stream().map(v -> v.isSnapshot() ? v.withoutQualifier() : v).forEach(versions::add);
 
-        final List<String> includeVersions = versions.stream()
-            .map(
-                version -> version.isBefore(QualifiedVersion.of("7.17.0")) && version.hasQualifier() == false
-                    ? version.major() + "." + version.minor()
-                    : version.toString()
-            )
-            .distinct()
-            .collect(Collectors.toList());
+        final List<String> includeVersions = versions.stream().map(QualifiedVersion::toString).collect(Collectors.toList());
 
         final Map<String, Object> bindings = new HashMap<>();
         bindings.put("versions", versions);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.internal.precommit.ValidateYamlAgainstSchemaTask;
+import org.gradle.api.Action;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
@@ -22,6 +23,7 @@
 import org.gradle.api.tasks.util.PatternSet;
 
 import java.io.File;
+import java.util.function.Function;
 
 import javax.inject.Inject;
 
@@ -67,10 +69,14 @@ public void apply(Project project) {
                 task.dependsOn(validateChangelogsAgainstYamlTask);
             });
 
-        project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(task -> {
+        final Function<Boolean, Action<GenerateReleaseNotesTask>> configureGenerateTask = shouldConfigureYamlFiles -> task -> {
             task.setGroup("Documentation");
-            task.setDescription("Generates release notes from changelog files held in this checkout");
-            task.setChangelogs(yamlFiles);
+            if (shouldConfigureYamlFiles) {
+                task.setChangelogs(yamlFiles);
+                task.setDescription("Generates release notes from changelog files held in this checkout");
+            } else {
+                task.setDescription("Generates stub release notes e.g. after feature freeze");
+            }
 
             task.setReleaseNotesIndexTemplate(projectDirectory.file(RESOURCES + "templates/release-notes-index.asciidoc"));
             task.setReleaseNotesIndexFile(projectDirectory.file("docs/reference/release-notes.asciidoc"));
@@ -96,9 +102,16 @@ public void apply(Project project) {
                     String.format("docs/reference/migration/migrate_%d_%d.asciidoc", version.getMajor(), version.getMinor())
                 )
             );
+            task.setMigrationIndexTemplate(projectDirectory.file(RESOURCES + "templates/migration-index.asciidoc"));
+            task.setMigrationIndexFile(projectDirectory.file("docs/reference/migration/index.asciidoc"));
 
             task.dependsOn(validateChangelogsTask);
-        });
+        };
+
+        project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(configureGenerateTask.apply(true));
+        project.getTasks()
+            .register("generateStubReleaseNotes", GenerateReleaseNotesTask.class)
+            .configure(configureGenerateTask.apply(false));
 
         project.getTasks().register("pruneChangelogs", PruneChangelogsTask.class).configure(task -> {
             task.setGroup("Documentation");