Rework the handling of profiles

- do not embed any static profile
- compute the java packages based on the running vm and toolchains
- fall back to "higher" profiles if an exact match is not found
- do not validate BREEs against static list of names
- fix java 9 compatibility of ListSystemPackages

RELEASE_NOTES.md

@@ -4,6 +4,11 @@ This page describes the noteworthy improvements provided by each release of Ecli
 
 ## 3.0.0 (under development)
 
+### Tycho no longer ships JVM profiles
+
+Because of modular VMs the profiles shipped by Tycho has never been complete and actually are already partly generated in regards to available packages.
+From now on, Tycho do not ship any profiles and thus you can use any VM in the toolchains or as a running VM and Tycho will generate a profile for it.
+
 ### Enhanced Support for Maven CI Friendly Versions
 
 Starting with Maven 3.8.5 Tycho now supports an enhanced form of the [Maven CI Friendly Versions](https://maven.apache.org/maven-ci-friendly.html) beside the standard properties names one could also use:
@@ -186,6 +191,11 @@ This can be useful if you like to execute the build with multiple threads (e.g.
 
 ### Migration guide 2.x -> 3.x
 
+#### publish-osgi-ee do not publish a fixed size of profiles anymore
+
+The `publish-osgi-ee` previously has published a fixes list of "usefull" execution environments maintained by Tycho.
+This is no longer true and Tycho do publish only those JavaSE profiles that are available to the build and have a version larger than 11 if not configured explicitly.
+
 #### jgit-timestamp provider moved from `org.eclipse.tycho.extras` to `org.eclipse.tycho`
 
 The `tycho-buildtimestamp-jgit` plugin has moved to the `org.eclipse.tycho` group id.

tycho-compiler-plugin/src/main/java/org/eclipse/tycho/compiler/AbstractOsgiCompilerMojo.java

@@ -392,7 +392,8 @@ public StandardExecutionEnvironment[] getBREE() {
                                     return null;
                                 }
                             }).filter(Objects::nonNull).collect(Collectors.toList());
-                    manifestBREEs = ExecutionEnvironmentUtils.getProfileNames().stream() //
+                    manifestBREEs = ExecutionEnvironmentUtils.getProfileNames(toolchainManager, session, logger)
+                            .stream() //
                             .map(name -> name.split("-")) //
                             .map(segments -> Map.of(ExecutionEnvironmentNamespace.EXECUTION_ENVIRONMENT_NAMESPACE,
                                     segments[0], "version", segments[1]))

tycho-core/src/main/java/org/eclipse/tycho/core/ee/ExecutionEnvironmentConfigurationImpl.java

@@ -106,7 +106,9 @@ public boolean isCustomProfile() {
             return false;
         }
         String profileName = getProfileName();
-        boolean profileExists = ExecutionEnvironmentUtils.getProfileNames().contains(profileName);
+        Collection<String> profileNames = ExecutionEnvironmentUtils.getProfileNames(toolchainManager, session, logger);
+        boolean profileExists = profileNames
+                .contains(profileName);
         if (!profileExists && ignoredByResolver) {
             throw new BuildFailureException(
                     "When using a custom execution environment profile, resolveWithExecutionEnvironmentConstraints must not be set to false");
@@ -162,7 +164,7 @@ public boolean isIgnoredByResolver() {
 
     @Override
     public Collection<ExecutionEnvironment> getAllKnownEEs() {
-        return ExecutionEnvironmentUtils.getProfileNames().stream() //
+        return ExecutionEnvironmentUtils.getProfileNames(toolchainManager, session, logger).stream() //
                 .map(profileName -> ExecutionEnvironmentUtils.getExecutionEnvironment(profileName, toolchainManager,
                         session, logger)) //
                 .collect(Collectors.toList());

tycho-core/src/main/java/org/eclipse/tycho/core/ee/ExecutionEnvironmentUtils.java

@@ -14,20 +14,20 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
-import java.util.LinkedHashMap;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 import org.apache.maven.execution.MavenSession;
 import org.apache.maven.toolchain.Toolchain;
 import org.apache.maven.toolchain.ToolchainManager;
 import org.codehaus.plexus.logging.Logger;
 import org.eclipse.osgi.internal.framework.EquinoxConfiguration;
+import org.eclipse.tycho.core.ee.StandardExecutionEnvironment.JavaInfo;
 import org.eclipse.tycho.core.ee.shared.ExecutionEnvironment;
 import org.eclipse.tycho.core.ee.shared.ExecutionEnvironment.SystemPackageEntry;
 import org.osgi.framework.BundleActivator;
@@ -41,23 +41,7 @@
  */
 public class ExecutionEnvironmentUtils {
 
-    private static final Map<String, Properties> profilesProperties = fillEnvironmentsMap();
-    private static final Map<String, StandardExecutionEnvironment> executionEnvironmentsMap = new ConcurrentHashMap<>(
-            profilesProperties.size(), 1.f);
-
-    private static Map<String, Properties> fillEnvironmentsMap() {
-        Properties listProps = readProperties(findInSystemBundle("profile.list"));
-        List<String> profileFiles = new ArrayList<>(Arrays.asList(listProps.getProperty("java.profiles").split(",")));
-        profileFiles.add("JavaSE-11.profile");
-        profileFiles.add("JavaSE-17.profile");
-        profileFiles.add("JavaSE-18.profile");
-        Map<String, Properties> envMap = new LinkedHashMap<>(profileFiles.size(), 1.f);
-        for (String profileFile : profileFiles) {
-            Properties props = readProperties(findInSystemBundle(profileFile.trim()));
-            envMap.put(props.getProperty(EquinoxConfiguration.PROP_OSGI_JAVA_PROFILE_NAME).trim(), props);
-        }
-        return envMap;
-    }
+    private static Map<String, StandardExecutionEnvironment> executionEnvironmentsMap;
 
     private static Properties readProperties(final URL url) {
         Properties listProps = new Properties();
@@ -81,20 +65,104 @@ private static Properties readProperties(final URL url) {
      */
     public static StandardExecutionEnvironment getExecutionEnvironment(String profileName, ToolchainManager manager,
             MavenSession session, Logger logger) throws UnknownEnvironmentException {
-        if (!profilesProperties.containsKey(profileName)) {
-            throw new UnknownEnvironmentException(profileName);
+        Map<String, StandardExecutionEnvironment> map = getExecutionEnvironmentsMap(manager, session, logger);
+        StandardExecutionEnvironment ee = map.get(profileName);
+        if (ee != null) {
+            return ee;
+        }
+        int version = getVersion(profileName);
+        if (version > 8) {
+            //try find newer version...
+            StandardExecutionEnvironment higherEE = map.keySet().stream()
+                    .mapToInt(ExecutionEnvironmentUtils::getVersion).filter(v -> v > version).min().stream()
+                    .mapToObj(v -> {
+                        String[] split = profileName.split("-");
+                        return split[0] + "-" + v;
+                    }).map(map::get).findFirst().orElse(null);
+            if (higherEE != null) {
+                logger.warn("Using " + higherEE.getProfileName() + " to fulfill requested profile of " + profileName
+                        + " this might lead to faulty dependency resolution, consider define a suitable VM in the toolchains.");
+                return higherEE;
+            }
+        }
+        throw new UnknownEnvironmentException(profileName);
+    }
+
+    public static Collection<String> getProfileNames(ToolchainManager manager, MavenSession session, Logger logger) {
+
+        return new ArrayList<>(getExecutionEnvironmentsMap(manager, session, logger).keySet());
+    }
+
+    private static synchronized Map<String, StandardExecutionEnvironment> getExecutionEnvironmentsMap(
+            ToolchainManager manager, MavenSession session, Logger logger) {
+        if (executionEnvironmentsMap == null) {
+            executionEnvironmentsMap = new HashMap<String, StandardExecutionEnvironment>();
+            Properties listProps = readProperties(findInSystemBundle("profile.list"));
+            //first read all profiles that are part of the system...
+            for (String profileFile : listProps.getProperty("java.profiles").split(",")) {
+                Properties props = readProperties(findInSystemBundle(profileFile.trim()));
+                if (props == null) {
+                    logger.warn("can't read profile " + profileFile + " from system path");
+                    continue;
+                }
+                String name = props.getProperty(EquinoxConfiguration.PROP_OSGI_JAVA_PROFILE_NAME).trim();
+                executionEnvironmentsMap.put(name,
+                        new StandardExecutionEnvironment(props, getToolchainFor(name, manager, session), logger));
+            }
+            //derive from the toolchains...
+            if (manager != null) {
+                List<Toolchain> jdks = manager.getToolchains(session, "jdk", null);
+                for (Toolchain jdk : jdks) {
+                    JavaInfo javaInfo = StandardExecutionEnvironment.readFromToolchains(jdk, logger);
+                    if (javaInfo.version > 8) {
+                        Properties toolchainJvm = createProfileJvm(javaInfo.version, javaInfo.packages);
+                        String name = toolchainJvm.getProperty(EquinoxConfiguration.PROP_OSGI_JAVA_PROFILE_NAME).trim();
+                        executionEnvironmentsMap.put(name, new StandardExecutionEnvironment(toolchainJvm, jdk, logger));
+                    }
+                }
+            }
+            //derive from the running jvm...
+            int javaVersion = Runtime.version().feature();
+            if (!executionEnvironmentsMap.containsKey("JavaSE-" + javaVersion)) {
+                Properties runningVm = createProfileJvm(javaVersion, ListSystemPackages.getCurrentJREPackages());
+                String name = runningVm.getProperty(EquinoxConfiguration.PROP_OSGI_JAVA_PROFILE_NAME).trim();
+                executionEnvironmentsMap.put(name,
+                        new StandardExecutionEnvironment(runningVm, getToolchainFor(name, manager, session), logger));
+            }
         }
-        return executionEnvironmentsMap.computeIfAbsent(profileName, name -> {
-            List<Toolchain> toolchains = manager != null && session != null
-                    ? manager.getToolchains(session, "jdk", Collections.singletonMap("id", profileName))
-                    : Collections.emptyList();
-            return new StandardExecutionEnvironment(profilesProperties.get(name),
-                    toolchains.isEmpty() ? null : toolchains.iterator().next(), logger);
-        });
+        return executionEnvironmentsMap;
     }
 
-    public static List<String> getProfileNames() {
-        return new ArrayList<>(profilesProperties.keySet());
+    private static Toolchain getToolchainFor(String profileName, ToolchainManager manager, MavenSession session) {
+        if (manager != null) {
+            //First try to find it by ID
+            for (Toolchain toolchain : manager.getToolchains(session, "jdk",
+                    Collections.singletonMap("id", profileName))) {
+                return toolchain;
+            }
+            //Try find by version
+            int version = getVersion(profileName);
+            if (version > 8) {
+                for (Toolchain toolchain : manager.getToolchains(session, "jdk",
+                        Collections.singletonMap("version", String.valueOf(version)))) {
+                    return toolchain;
+                }
+            }
+        }
+        return null;
+    }
+
+    public static int getVersion(String profileName) {
+        String[] split = profileName.split("-");
+        if (split.length == 2) {
+            try {
+                return (int) Double.parseDouble(split[split.length - 1]);
+            } catch (NumberFormatException e) {
+                //can't check then...
+            }
+        }
+        return -1;
+
     }
 
     public static void applyProfileProperties(Properties properties, ExecutionEnvironment executionEnvironment) {
@@ -137,4 +205,45 @@ private static URL findInSystemBundle(String entry) {
         ClassLoader loader = BundleActivator.class.getClassLoader();
         return loader == null ? ClassLoader.getSystemResource(entry) : loader.getResource(entry);
     }
+
+    //This is derived from org.eclipse.equinox.p2.publisher.actions.JREAction.createDefaultProfileFromRunningJvm()
+    private static Properties createProfileJvm(int javaVersion, Collection<String> packages) {
+        String profileName = "JavaSE-" + javaVersion;
+        Properties props = new Properties();
+        // add systempackages
+        props.setProperty("org.osgi.framework.system.packages", packages.stream().collect(Collectors.joining(",")));
+        // add EE
+        StringBuilder ee = new StringBuilder(
+                "OSGi/Minimum-1.0,OSGi/Minimum-1.1,OSGi/Minimum-1.2,JavaSE/compact1-1.8,JavaSE/compact2-1.8,JavaSE/compact3-1.8,JRE-1.1,J2SE-1.2,J2SE-1.3,J2SE-1.4,J2SE-1.5,JavaSE-1.6,JavaSE-1.7,JavaSE-1.8,");
+        for (int i = 9; i < javaVersion; i++) {
+            ee.append("JavaSE-" + String.valueOf(i) + ",");
+        }
+        ee.append(profileName);
+        props.setProperty("org.osgi.framework.executionenvironment", ee.toString());
+        // add capabilities
+        StringBuilder versionList = new StringBuilder();
+        versionList.append("1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8");
+        for (int i = 9; i <= javaVersion; i++) {
+            versionList.append(", " + String.valueOf(i) + ".0");
+        }
+        props.setProperty("org.osgi.framework.system.capabilities",
+                "osgi.ee; osgi.ee=\"OSGi/Minimum\"; version:List<Version>=\"1.0, 1.1, 1.2\",osgi.ee; osgi.ee=\"JRE\"; version:List<Version>=\"1.0, 1.1\",osgi.ee; osgi.ee=\"JavaSE\"; version:List<Version>=\""
+                        + versionList.toString()
+                        + "\",osgi.ee; osgi.ee=\"JavaSE/compact1\"; version:List<Version>=\"1.8,"
+                        + String.valueOf(javaVersion)
+                        + ".0\",osgi.ee; osgi.ee=\"JavaSE/compact2\"; version:List<Version>=\"1.8,"
+                        + String.valueOf(javaVersion)
+                        + ".0\",osgi.ee; osgi.ee=\"JavaSE/compact3\"; version:List<Version>=\"1.8,"
+                        + String.valueOf(javaVersion) + ".0\"");
+
+        // add profile name and compiler options
+        props.setProperty("osgi.java.profile.name", profileName);
+        props.setProperty("org.eclipse.jdt.core.compiler.compliance", String.valueOf(javaVersion));
+        props.setProperty("org.eclipse.jdt.core.compiler.source", String.valueOf(javaVersion));
+        props.setProperty("org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode", "enabled");
+        props.setProperty("org.eclipse.jdt.core.compiler.codegen.targetPlatform", String.valueOf(javaVersion));
+        props.setProperty("org.eclipse.jdt.core.compiler.problem.assertIdentifier", "error");
+        props.setProperty("org.eclipse.jdt.core.compiler.problem.enumIdentifier", "error");
+        return props;
+    }
 }

tycho-core/src/main/java/org/eclipse/tycho/core/ee/ListSystemPackages.java

@@ -7,6 +7,8 @@
  *******************************************************************************/
 package org.eclipse.tycho.core.ee;
 
+import java.lang.Runtime.Version;
+import java.lang.module.ModuleDescriptor.Exports;
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -19,17 +21,27 @@
 public class ListSystemPackages {
 
     public static void main(String[] args) {
+        Version version = Runtime.version();
+        try {
+            System.out.println(version.feature());
+        } catch (NoSuchMethodError e) {
+            System.out.println("9");
+        }
         getCurrentJREPackages().forEach(System.out::println);
     }
 
     public static Set<String> getCurrentJREPackages() {
         return ModuleLayer.boot().modules().stream().map(Module::getDescriptor) //
                 .flatMap(desc -> desc.isAutomatic() ? //
                         desc.packages().stream() : //
-                        desc.exports().stream()
-                                .filter(Predicate.not(java.lang.module.ModuleDescriptor.Exports::isQualified))
+                        desc.exports().stream().filter(not(java.lang.module.ModuleDescriptor.Exports::isQualified))
                                 .map(java.lang.module.ModuleDescriptor.Exports::source) //
                 ).collect(Collectors.toSet());
     }
 
+    private static Predicate<? super Exports> not(Predicate<Exports> predicate) {
+        // java 9 compatible polyfill
+        return x -> !predicate.test(x);
+    }
+
 }

tycho-core/src/main/java/org/eclipse/tycho/core/ee/StandardExecutionEnvironment.java

@@ -110,29 +110,49 @@ public class StandardExecutionEnvironment implements Comparable<StandardExecutio
         this.logger = logger;
     }
 
-    private Set<String> readFromToolchains(Toolchain toolchain) {
+    static JavaInfo readFromToolchains(Toolchain toolchain, Logger logger) {
         if (toolchain == null) {
-            return Collections.emptySet();
+            return new JavaInfo(-1, Collections.emptySet());
         }
         String java = toolchain.findTool("java");
         if (java == null) {
-            return Collections.emptySet();
+            return new JavaInfo(-1, Collections.emptySet());
         }
         Set<String> res = new HashSet<>();
+        int version = -1;
         try {
             ProcessBuilder builder = new ProcessBuilder(java, "-jar",
                     getSystemPackagesCompanionJar().getAbsolutePath());
             try (BufferedReader reader = new BufferedReader(
                     new java.io.InputStreamReader(builder.start().getInputStream(), Charset.defaultCharset()))) {
-                String line = null;
+                String line = reader.readLine();
+                try {
+                    if (line != null) {
+                        //for old vms < java 9 we might get no response at all
+                        version = Integer.parseInt(line);
+                    }
+                } catch (NumberFormatException e) {
+                    logger.error(e.getMessage(), e);
+                }
                 while ((line = reader.readLine()) != null) {
                     res.add(line);
                 }
             }
         } catch (IOException e) {
             logger.error(e.getMessage(), e);
         }
-        return res;
+        return new JavaInfo(version, res);
+    }
+
+    static final class JavaInfo {
+        final int version;
+        final Collection<String> packages;
+
+        private JavaInfo(int version, Collection<String> packages) {
+            this.version = version;
+            this.packages = Collections.unmodifiableCollection(packages);
+        }
+
     }
 
     static File getSystemPackagesCompanionJar() throws IOException {
@@ -226,7 +246,7 @@ public synchronized Collection<SystemPackageEntry> getSystemPackages() {
             } else if (toolchain != null) {
                 logger.debug(
                         "No system.packages in profile definition file for " + profileName + "; checking toolchain.");
-                this.systemPackages = readFromToolchains(toolchain).stream()
+                this.systemPackages = readFromToolchains(toolchain, logger).packages.stream()
                         .map(packageName -> new SystemPackageEntry(packageName, null)).collect(Collectors.toList());
             } else if (Integer.parseInt(compilerSourceLevel) == Runtime.version().feature()) {
                 logger.debug("Currently running JRE matches source level for " + getProfileName()

tycho-core/src/main/java/org/eclipse/tycho/core/osgitools/EquinoxResolver.java

@@ -242,7 +242,7 @@ protected Properties getPlatformProperties(Properties properties, MavenSession m
             // ignoring EE by adding all known EEs
             StringJoiner allSystemPackages = new StringJoiner(",");
             StringJoiner allSystemCapabilities = new StringJoiner(",");
-            for (String profile : ExecutionEnvironmentUtils.getProfileNames()) {
+            for (String profile : ExecutionEnvironmentUtils.getProfileNames(toolchainManager, mavenSession, logger)) {
                 StandardExecutionEnvironment executionEnvironment = ExecutionEnvironmentUtils
                         .getExecutionEnvironment(profile, toolchainManager, mavenSession, logger);
                 String currentSystemPackages = (String) executionEnvironment.getProfileProperties()