Skip to content

Commit

Permalink
Scripting: enforce GPLv2 for parsed stdlib docs (elastic#68601)
Browse files Browse the repository at this point in the history 
When parsing Java standard library javadocs, we need to ensure
that our use will comply with the license.  Our use complies
with GPLv2 licenses but may not comply with proprietary licenses.

Reject .java files that have non-GPL licenses when parsing them
for parameter names and javadoc comments.
  • Loading branch information
@stu-elastic @easyice
stu-elastic authored and easyice committed Mar 25, 2021
1 parent a3963e6 commit 6a5fa78
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 39 deletions.
39 changes: 35 additions & 4 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextApiSpecGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.painless.action.PainlessContextInfo;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
Expand All @@ -28,9 +30,9 @@ public static void main(String[] args) throws IOException {
List<PainlessContextInfo> contexts = ContextGeneratorCommon.getContextInfos();
Path rootDir = resetRootDir();
ContextGeneratorCommon.PainlessInfos infos;
Path jdksrc = getJdkSrc();
JavaClassFilesystemResolver jdksrc = getJdkSrc();
if (jdksrc != null) {
infos = new ContextGeneratorCommon.PainlessInfos(contexts, new StdlibJavadocExtractor(jdksrc));
infos = new ContextGeneratorCommon.PainlessInfos(contexts, new JavadocExtractor(jdksrc));
} else {
infos = new ContextGeneratorCommon.PainlessInfos(contexts);
}
Expand Down Expand Up @@ -70,11 +72,40 @@ private static Path resetRootDir() throws IOException {
}

@SuppressForbidden(reason = "resolve jdk src directory with environment")
private static Path getJdkSrc() {
private static JavaClassFilesystemResolver getJdkSrc() {
String jdksrc = System.getProperty("jdksrc");
if (jdksrc == null || "".equals(jdksrc)) {
return null;
}
return PathUtils.get(jdksrc);
return new JavaClassFilesystemResolver(PathUtils.get(jdksrc));
}

public static class JavaClassFilesystemResolver implements JavaClassResolver {
private final Path root;

public JavaClassFilesystemResolver(Path root) {
this.root = root;
}

@SuppressForbidden(reason = "resolve class file from java src directory with environment")
public InputStream openClassFile(String className) throws IOException {
// TODO(stu): handle primitives & not stdlib
if (className.contains(".") && className.startsWith("java")) {
int dollarPosition = className.indexOf("$");
if (dollarPosition >= 0) {
className = className.substring(0, dollarPosition);
}
String[] packages = className.split("\\.");
String path = String.join("/", packages);
Path classPath = root.resolve(path + ".java");
return new FileInputStream(classPath.toFile());
}
return new InputStream() {
@Override
public int read() throws IOException {
return -1;
}
};
}
}
}
2 changes: 1 addition & 1 deletion modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextGeneratorCommon.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ public PainlessInfos(List<PainlessContextInfo> contextInfos) {
.collect(Collectors.toList());
}

public PainlessInfos(List<PainlessContextInfo> contextInfos, StdlibJavadocExtractor extractor) throws IOException {
public PainlessInfos(List<PainlessContextInfo> contextInfos, JavadocExtractor extractor) throws IOException {
javaNamesToDisplayNames = getDisplayNames(contextInfos);

javaNamesToJavadoc = new HashMap<>();
Expand Down
16 changes: 16 additions & 0 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavaClassResolver.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.painless;

import java.io.IOException;
import java.io.InputStream;

public interface JavaClassResolver {
InputStream openClassFile(String className) throws IOException;
}
75 changes: 49 additions & 26 deletions ...arch/painless/StdlibJavadocExtractor.java → ...sticsearch/painless/JavadocExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,57 +15,65 @@
import com.github.javaparser.ast.body.FieldDeclaration;
import com.github.javaparser.ast.body.MethodDeclaration;
import com.github.javaparser.ast.body.VariableDeclarator;
import com.github.javaparser.ast.comments.Comment;
import com.github.javaparser.ast.visitor.VoidVisitorAdapter;
import com.github.javaparser.javadoc.Javadoc;
import org.elasticsearch.common.SuppressForbidden;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

public class StdlibJavadocExtractor {
private final Path root;
public class JavadocExtractor {

public StdlibJavadocExtractor(Path root) {
this.root = root;
}
private final JavaClassResolver resolver;

@SuppressForbidden(reason = "resolve class file from java src directory with environment")
private File openClassFile(String className) {
int dollarPosition = className.indexOf("$");
if (dollarPosition >= 0) {
className = className.substring(0, dollarPosition);
}
String[] packages = className.split("\\.");
String path = String.join("/", packages);
Path classPath = root.resolve(path + ".java");
return classPath.toFile();
private static final String GPLv2 = "* This code is free software; you can redistribute it and/or modify it"+
"\n * under the terms of the GNU General Public License version 2 only, as"+
"\n * published by the Free Software Foundation.";

public JavadocExtractor(JavaClassResolver resolver) {
this.resolver = resolver;
}


public ParsedJavaClass parseClass(String className) throws IOException {
ParsedJavaClass pj = new ParsedJavaClass();
if (className.contains(".") && className.startsWith("java")) { // TODO(stu): handle primitives & not stdlib
ClassFileVisitor visitor = new ClassFileVisitor();
CompilationUnit cu = StaticJavaParser.parse(openClassFile(className));
visitor.visit(cu, pj);
}
return pj;
InputStream classStream = resolver.openClassFile(className);
ParsedJavaClass parsed = new ParsedJavaClass(GPLv2);
if (classStream == null) {
return parsed;
}
ClassFileVisitor visitor = new ClassFileVisitor();
CompilationUnit cu = StaticJavaParser.parse(classStream);
visitor.visit(cu, parsed);
return parsed;
}

public static class ParsedJavaClass {
public final Map<MethodSignature, ParsedMethod> methods;
public final Map<String, String> fields;
public final Map<List<String>, ParsedMethod> constructors;
private final String license;
private boolean valid = false;
private boolean validated = false;

public ParsedJavaClass() {
public ParsedJavaClass(String license) {
methods = new HashMap<>();
fields = new HashMap<>();
constructors = new HashMap<>();
this.license = license;
}

public void validateLicense(Optional<Comment> license) {
if (validated) {
throw new IllegalStateException("Cannot double validate the license");
}
this.valid = license.map(Comment::getContent).orElse("").contains(this.license);
validated = true;
}

public ParsedMethod getMethod(String name, List<String> parameterTypes) {
Expand All @@ -80,6 +88,9 @@ public String toString() {
}

public void putMethod(MethodDeclaration declaration) {
if (valid == false) {
return;
}
methods.put(
MethodSignature.fromDeclaration(declaration),
new ParsedMethod(
Expand All @@ -93,6 +104,9 @@ public void putMethod(MethodDeclaration declaration) {
}

public void putConstructor(ConstructorDeclaration declaration) {
if (valid == false) {
return;
}
constructors.put(
declaration.getParameters().stream().map(p -> stripTypeParameters(p.getType().asString())).collect(Collectors.toList()),
new ParsedMethod(
Expand Down Expand Up @@ -134,6 +148,9 @@ public String getField(String name) {
}

public void putField(FieldDeclaration declaration) {
if (valid == false) {
return;
}
for (VariableDeclarator var : declaration.getVariables()) {
fields.put(var.getNameAsString(), declaration.getJavadoc().map(Javadoc::toText).orElse(""));
}
Expand Down Expand Up @@ -185,6 +202,12 @@ public ParsedMethod(String javadoc, List<String> parameterNames) {
}

private static class ClassFileVisitor extends VoidVisitorAdapter<ParsedJavaClass> {
@Override
public void visit(CompilationUnit compilationUnit, ParsedJavaClass parsed) {
parsed.validateLicense(compilationUnit.getComment());
super.visit(compilationUnit, parsed);
}

@Override
public void visit(MethodDeclaration methodDeclaration, ParsedJavaClass parsed) {
parsed.putMethod(methodDeclaration);
Expand Down
16 changes: 8 additions & 8 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public Context(
PainlessContextInfo info,
Set<PainlessContextClassInfo> commonClassInfos,
Map<String, String> javaNamesToDisplayNames,
StdlibJavadocExtractor extractor
JavadocExtractor extractor
) throws IOException {
this.name = info.getName();
List<PainlessContextClassInfo> classInfos = ContextGeneratorCommon.excludeCommonClassInfos(commonClassInfos, info.getClasses());
Expand Down Expand Up @@ -111,11 +111,11 @@ private Class(
public static List<Class> fromInfos(
List<PainlessContextClassInfo> infos,
Map<String, String> javaNamesToDisplayNames,
StdlibJavadocExtractor extractor
JavadocExtractor extractor
) throws IOException {
List<Class> classes = new ArrayList<>(infos.size());
for (PainlessContextClassInfo info : infos) {
StdlibJavadocExtractor.ParsedJavaClass parsedClass = extractor.parseClass(info.getName());
JavadocExtractor.ParsedJavaClass parsedClass = extractor.parseClass(info.getName());
classes.add(new Class(
javaNamesToDisplayNames.get(info.getName()),
info.isImported(),
Expand Down Expand Up @@ -206,7 +206,7 @@ public static List<Method> fromInfos(List<PainlessContextMethodInfo> infos, Map<
public static List<Method> fromInfos(
List<PainlessContextMethodInfo> infos,
Map<String, String> javaNamesToDisplayNames,
StdlibJavadocExtractor.ParsedJavaClass parsed
JavadocExtractor.ParsedJavaClass parsed
) {
List<Method> methods = new ArrayList<>(infos.size());
for (PainlessContextMethodInfo info: infos) {
Expand All @@ -216,7 +216,7 @@ public static List<Method> fromInfos(
String name = info.getName();
List<String> parameterTypes = info.getParameters();

StdlibJavadocExtractor.ParsedMethod parsedMethod = parsed.getMethod(name, parameterTypes);
JavadocExtractor.ParsedMethod parsedMethod = parsed.getMethod(name, parameterTypes);
if (parsedMethod != null) {
javadoc = parsedMethod.javadoc;
parameterNames = parsedMethod.parameterNames;
Expand Down Expand Up @@ -285,15 +285,15 @@ public static List<Constructor> fromInfos(List<PainlessContextConstructorInfo> i
private static List<Constructor> fromInfos(
List<PainlessContextConstructorInfo> infos,
Map<String, String> javaNamesToDisplayNames,
StdlibJavadocExtractor.ParsedJavaClass pj
JavadocExtractor.ParsedJavaClass pj
) {
List<Constructor> constructors = new ArrayList<>(infos.size());
for (PainlessContextConstructorInfo info: infos) {
List<String> parameterTypes = info.getParameters();
List<String> parameterNames = null;
String javadoc = null;

StdlibJavadocExtractor.ParsedMethod parsed = pj.getConstructor(parameterTypes);
JavadocExtractor.ParsedMethod parsed = pj.getConstructor(parameterTypes);
if (parsed != null) {
parameterNames = parsed.parameterNames;
javadoc = parsed.javadoc;
Expand Down Expand Up @@ -357,7 +357,7 @@ public static List<Field> fromInfos(List<PainlessContextFieldInfo> infos, Map<St
public static List<Field> fromInfos(
List<PainlessContextFieldInfo> infos,
Map<String, String> javaNamesToDisplayNames,
StdlibJavadocExtractor.ParsedJavaClass pj
JavadocExtractor.ParsedJavaClass pj
) {
List<Field> fields = new ArrayList<>(infos.size());
for (PainlessContextFieldInfo info: infos) {
Expand Down

0 comments on commit 6a5fa78

Please sign in to comment.