From 37dcb02d3556ac9e2199d11b82538b21ad99e6ff Mon Sep 17 00:00:00 2001 From: Junchuan Wang Date: Mon, 24 Apr 2023 01:19:05 -0700 Subject: [PATCH] Upgrade javadoc apis --- .../tools/idlgen/DocletDocsProvider.java | 188 +++++----- .../restli/tools/idlgen/RestLiDoclet.java | 331 +++++++++++------- 2 files changed, 299 insertions(+), 220 deletions(-) diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java index f0cd6ecc41..496915615d 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java @@ -21,27 +21,37 @@ import com.linkedin.restli.server.annotations.ActionParam; import com.linkedin.restli.server.annotations.QueryParam; +import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Method; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; -import com.sun.javadoc.AnnotationDesc; -import com.sun.javadoc.ClassDoc; -import com.sun.javadoc.Doc; -import com.sun.javadoc.MethodDoc; -import com.sun.javadoc.ParamTag; -import com.sun.javadoc.Parameter; -import com.sun.javadoc.Tag; +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ReturnTree; import org.apache.commons.io.output.NullWriter; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; + /** * Specialized {@link DocsProvider} whose documentation comes from the Javadoc Doclet {@link RestLiDoclet}. @@ -76,6 +86,33 @@ public Set supportedFileExtensions() return Collections.singleton(".java"); } + public static List collectSourceFiles(List sourcePaths, Collection packageNames) throws IOException { + List sourceFiles = new ArrayList<>(); + for (String sourcePath : sourcePaths) { + Path basePath = Paths.get(sourcePath); + Files.walkFileTree(basePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(".java")) { + if (packageNames == null || packageNames.isEmpty()) { + sourceFiles.add(file.toString()); + } else { + String packageName = basePath.relativize(file.getParent()).toString().replace('/', '.'); + for (String targetPackageName : packageNames) { + if (packageName.startsWith(targetPackageName)) { + sourceFiles.add(file.toString()); + break; + } + } + } + } + return FileVisitResult.CONTINUE; + } + }); + } + return sourceFiles; + } + @Override public void registerSourceFiles(Collection sourceFileNames) { @@ -93,89 +130,86 @@ public void registerSourceFiles(Collection sourceFileNames) final PrintWriter sysoutWriter = new PrintWriter(System.out, true); final PrintWriter nullWriter = new PrintWriter(new NullWriter()); - final List javadocArgs = new ArrayList<>(Arrays.asList("-classpath", - flatClasspath, - "-sourcepath", - StringUtils.join(_sourcePaths, ":"))); - if (_resourcePackages != null) + + + List analyzedSourceFiles; + try { - javadocArgs.add("-subpackages"); - javadocArgs.add(StringUtils.join(_resourcePackages, ":")); + analyzedSourceFiles = new ArrayList<>(collectSourceFiles(Arrays.asList(_sourcePaths), + _resourcePackages == null + ? null : Arrays.asList(_resourcePackages))); } - else + catch (IOException e) + { + throw new RuntimeException("Failed to collect source files", e); + } + + + if (_resourcePackages == null) { - javadocArgs.addAll(sourceFileNames); + analyzedSourceFiles.addAll(sourceFileNames); } _doclet = RestLiDoclet.generateDoclet(_apiName, sysoutWriter, nullWriter, nullWriter, - javadocArgs.toArray(new String[0])); + flatClasspath, + analyzedSourceFiles + ); } @Override public String getClassDoc(Class resourceClass) { - final ClassDoc doc = _doclet.getClassDoc(resourceClass); + final TypeElement doc = _doclet.getClassDoc(resourceClass); if (doc == null) { return null; } - return buildDoc(doc.commentText()); + return buildDoc(_doclet.getDocComment(doc)); } - @Override - public String getClassDeprecatedTag(Class resourceClass) - { - final ClassDoc doc = _doclet.getClassDoc(resourceClass); - if (doc == null) - { + public String getClassDeprecatedTag(Class resourceClass) { + TypeElement typeElement = _doclet.getClassDoc(resourceClass); + if (typeElement == null) { return null; } - - return formatDeprecatedTags(doc); + return formatDeprecatedTags(typeElement); } - private static String formatDeprecatedTags(Doc doc) - { - Tag[] deprecatedTags = doc.tags("deprecated"); - if(deprecatedTags.length > 0) - { + private String formatDeprecatedTags(Element element) { + List deprecatedTags = _doclet.getDeprecatedTags(element); + if (!deprecatedTags.isEmpty()) { StringBuilder deprecatedText = new StringBuilder(); - for(int i = 0; i < deprecatedTags.length; i++) - { - deprecatedText.append(deprecatedTags[i].text()); - if(i < deprecatedTags.length - 1) - { + for (int i = 0; i < deprecatedTags.size(); i++) { + deprecatedText.append(deprecatedTags.get(i)); + if (i < deprecatedTags.size() - 1) { deprecatedText.append(" "); } } return deprecatedText.toString(); - } - else - { + } else { return null; } } - @Override public String getMethodDoc(Method method) { - final MethodDoc doc = _doclet.getMethodDoc(method); + final ExecutableElement doc = _doclet.getMethodDoc(method); if (doc == null) { return null; } - return buildDoc(doc.commentText()); + return buildDoc(_doclet.getDocComment(doc)); } @Override public String getMethodDeprecatedTag(Method method) { - final MethodDoc doc = _doclet.getMethodDoc(method); + final ExecutableElement doc = _doclet.getMethodDoc(method); if (doc == null) { return null; @@ -184,32 +218,28 @@ public String getMethodDeprecatedTag(Method method) return formatDeprecatedTags(doc); } + @Override public String getParamDoc(Method method, String name) { - final MethodDoc methodDoc = _doclet.getMethodDoc(method); + final ExecutableElement methodDoc = _doclet.getMethodDoc(method); if (methodDoc == null) { return null; } - for (Parameter parameter : methodDoc.parameters()) + for (VariableElement parameter : methodDoc.getParameters()) { - for (AnnotationDesc annotationDesc : parameter.annotations()) + for (AnnotationMirror annotationMirror : parameter.getAnnotationMirrors()) { - if (annotationDesc.isSynthesized()) - { - continue; - } - - if (isQueryParamAnnotation(annotationDesc) || isActionParamAnnotation(annotationDesc)) + if (isQueryParamAnnotation(annotationMirror) || isActionParamAnnotation(annotationMirror)) { - for (AnnotationDesc.ElementValuePair pair : annotationDesc.elementValues()) + for (Map.Entry entry : annotationMirror.getElementValues().entrySet()) { - if ("value".equals(pair.element().name()) && name.equals(pair.value().value())) + if ("value".equals(entry.getKey().getSimpleName().toString()) && name.equals(entry.getValue().getValue())) { - return getParamTagDoc(methodDoc, parameter.name()); + return getParamTagDoc(methodDoc, parameter.getSimpleName().toString()); } } } @@ -218,35 +248,23 @@ public String getParamDoc(Method method, String name) return null; } - - private static String getParamTagDoc(MethodDoc methodDoc, String name) - { - for (ParamTag tag : methodDoc.paramTags()) - { - if (name.equals(tag.parameterName())) - { - return buildDoc(tag.parameterComment()); - } - } - - return null; + private String getParamTagDoc(ExecutableElement methodElement, String name) { + Map paramTags = _doclet.getParamTags(methodElement); + return paramTags.get(name); } + @Override - public String getReturnDoc(Method method) - { - final MethodDoc methodDoc = _doclet.getMethodDoc(method); - if (methodDoc != null) - { - for (Tag tag : methodDoc.tags()) - { - if(tag.name().toLowerCase().equals("@return")) - { - return buildDoc(tag.text()); + public String getReturnDoc(Method method) { + ExecutableElement methodElement = _doclet.getMethodDoc(method); + if (methodElement != null) { + for (DocTree docTree : _doclet.getDocCommentTree(method).getFullBody()) { + if (docTree.getKind() == DocTree.Kind.RETURN) { + ReturnTree returnTree = (ReturnTree) docTree; + return buildDoc(returnTree.getDescription().toString()); } } } - return null; } @@ -260,14 +278,12 @@ private static String buildDoc(String docText) return null; } - - private static boolean isQueryParamAnnotation(AnnotationDesc annotationDesc) + private static boolean isQueryParamAnnotation(AnnotationMirror annotationMirror) { - return QueryParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); + return QueryParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); } - private static boolean isActionParamAnnotation(AnnotationDesc annotationDesc) + private static boolean isActionParamAnnotation(AnnotationMirror annotationMirror) { - return ActionParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); - } -} + return ActionParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + }} diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java index 025765a017..8a0404c53b 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java @@ -17,224 +17,287 @@ package com.linkedin.restli.tools.idlgen; -import com.sun.javadoc.ClassDoc; -import com.sun.javadoc.MethodDoc; -import com.sun.javadoc.Parameter; -import com.sun.javadoc.RootDoc; -import com.sun.javadoc.Type; -import com.sun.tools.javadoc.Main; +import jdk.javadoc.doclet.Doclet; +import jdk.javadoc.doclet.DocletEnvironment; +import jdk.javadoc.doclet.Reporter; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; - +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; + +import javax.lang.model.SourceVersion; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.TypeMirror; +import javax.lang.model.util.Elements; +import javax.tools.Diagnostic; +import javax.tools.DiagnosticListener; +import javax.tools.DocumentationTool; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; import java.io.PrintWriter; import java.lang.reflect.Method; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; -/** - * Custom Javadoc processor that merges documentation into the restspec.json. The embedded Javadoc - * generator is basically a commandline tool wrapper and it runs in complete isolation from the rest - * of the application. Due to the fact that the Javadoc tool instantiates RestLiDoclet, we cannot - * cleanly integrate the output into the {@link RestLiResourceModelExporter} tool. Thus, we're just - * dumping the docs into a static Map which can be accessed by {@link RestLiResourceModelExporter}. - * - * This class supports multiple runs of Javadoc Doclet API {@link Main#execute(String[])}. - * Each run will be assigned an unique "Doclet ID", returned by - * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String[])}. - * The Doclet ID should be subsequently used to initialize {@link DocletDocsProvider}. - * - * This class is thread-safe. However, #generateJavadoc() will be synchronized. - * - * @author dellamag - * @see Main#execute(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, String[]) - */ -public class RestLiDoclet +public class RestLiDoclet implements Doclet { private static RestLiDoclet _currentDocLet = null; private final DocInfo _docInfo; + private final DocletEnvironment _docEnv; + private final Elements _elements; + - /** - * Generate Javadoc and return associated Doclet ID. - * This method is synchronized. - * - * @param programName Name of the program (for error messages). - * @param errWriter PrintWriter to receive error messages. - * @param warnWriter PrintWriter to receive warning messages. - * @param noticeWriter PrintWriter to receive notice messages. - * @param args The command line parameters. - * @return an unique doclet ID which represent the subsequent Main#execute() run. - * @throws IllegalStateException if the generated doclet ID is already used. Try again. - * @throws IllegalArgumentException if Javadoc fails to generate docs. - */ public static synchronized RestLiDoclet generateDoclet(String programName, PrintWriter errWriter, PrintWriter warnWriter, PrintWriter noticeWriter, - String[] args) + String flatClassPath, + List analyzedSourceFiles + ) { - final int javadocRetCode = Main.execute(programName, errWriter, warnWriter, noticeWriter, RestLiDoclet.class.getName(), args); - if (javadocRetCode != 0) - { - throw new IllegalArgumentException("Javadoc failed with return code " + javadocRetCode); + noticeWriter.println("Generating Javadoc for " + programName); + + DocumentationTool docTool = ToolProvider.getSystemDocumentationTool(); + StandardJavaFileManager fileManager = docTool.getStandardFileManager(null, null, null); + Iterable fileObjects = fileManager.getJavaFileObjectsFromPaths( + analyzedSourceFiles.stream().map(Paths::get).collect(Collectors.toList())); + + + + + // Set up the Javadoc task options + List taskOptions = new ArrayList<>(); + taskOptions.add("-classpath"); + taskOptions.add(flatClassPath); + + // Create and run the Javadoc task + DocumentationTool.DocumentationTask task = docTool.getTask(errWriter, + fileManager, + new DiagnosticListener() { + @Override + public void report(Diagnostic diagnostic) { + switch (diagnostic.getKind()) { + case ERROR: + errWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case WARNING: + warnWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case NOTE: + noticeWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + } + } + }, + RestLiDoclet.class, + taskOptions, + fileObjects); + + boolean success = task.call(); + if (!success) { + throw new IllegalArgumentException("Javadoc generation failed"); } return _currentDocLet; } - /** - * Entry point for Javadoc Doclet. - * - * @param root {@link RootDoc} passed in by Javadoc - * @return is successful or not - */ - public static boolean start(RootDoc root) + private RestLiDoclet(DocInfo docInfo, DocletEnvironment docEnv) { - final DocInfo docInfo = new DocInfo(); + _docInfo = docInfo; + _docEnv = docEnv; + _elements = docEnv.getElementUtils(); + } + + public TypeElement getClassDoc(Class resourceClass) + { + return _docInfo.getClassDoc(resourceClass.getCanonicalName()); + } + + public ExecutableElement getMethodDoc(Method method) + { + final MethodIdentity methodId = MethodIdentity.create(method); + return _docInfo.getMethodDoc(methodId); + } - for (ClassDoc classDoc : root.classes()) - { - docInfo.setClassDoc(classDoc.qualifiedName(), classDoc); + public List getDeprecatedTags(Element element) { + List deprecatedTags = new ArrayList<>(); + Deprecated deprecatedAnnotation = element.getAnnotation(Deprecated.class); + if (deprecatedAnnotation != null) { + String deprecatedComment = _elements.getDocComment(element); + deprecatedTags.add(deprecatedComment); + } + return deprecatedTags; + } - for (MethodDoc methodDoc : classDoc.methods()) - { - docInfo.setMethodDoc(MethodIdentity.create(methodDoc), methodDoc); + public Map getParamTags(ExecutableElement method) { + Map paramTags = new HashMap<>(); + for (VariableElement parameter : method.getParameters()) { + String paramName = parameter.getSimpleName().toString(); + String paramComment = _elements.getDocComment(parameter); + if (paramComment != null) { + paramTags.put(paramName, paramComment); } } + return paramTags; + } - _currentDocLet = new RestLiDoclet(docInfo); + public com.sun.source.doctree.DocCommentTree getDocCommentTree(Method method) { + TypeElement typeElement = getClassDoc(method.getDeclaringClass()); + if (typeElement == null) { + return null; + } - return true; + for (Element element : typeElement.getEnclosedElements()) { + if (element.getSimpleName().toString().equals(method.getName())) { + return _docEnv.getDocTrees().getDocCommentTree(element); + } + } + + return null; } - private RestLiDoclet(DocInfo docInfo) - { - _docInfo = docInfo; + @Override + public void init(Locale locale, + Reporter reporter) { + // no-ops } - /** - * Query Javadoc {@link ClassDoc} for the specified resource class. - * - * @param resourceClass resource class to be queried - * @return corresponding {@link ClassDoc} - */ - public ClassDoc getClassDoc(Class resourceClass) - { - return _docInfo.getClassDoc(resourceClass.getCanonicalName()); + @Override + public String getName() { + return this.getClass().getSimpleName(); } - /** - * Query Javadoc {@link MethodDoc} for the specified Java method. - * - * @param method Java method to be queried - * @return corresponding {@link MethodDoc} - */ - public MethodDoc getMethodDoc(Method method) - { - final MethodIdentity methodId = MethodIdentity.create(method); - return _docInfo.getMethodDoc(methodId); + @Override + public Set getSupportedOptions() { + return Set.of(); } - private static class DocInfo - { - public ClassDoc getClassDoc(String className) - { - return _classNameToClassDoc.get(className); + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latest(); + } + + @Override + public boolean run(DocletEnvironment docEnv) { + final DocInfo docInfo = new DocInfo(); + + // Iterate through the TypeElements (class and interface declarations) + for (Element element : docEnv.getIncludedElements()) { + if (element instanceof TypeElement) { + TypeElement typeElement = (TypeElement) element; + docInfo.setClassDoc(typeElement.getQualifiedName().toString(), typeElement); + + // Iterate through the methods of the TypeElement + for (Element enclosedElement : typeElement.getEnclosedElements()) { + if (enclosedElement instanceof ExecutableElement) { + ExecutableElement methodElement = (ExecutableElement) enclosedElement; + docInfo.setMethodDoc(MethodIdentity.create(methodElement), methodElement); + } + } + } } - public MethodDoc getMethodDoc(MethodIdentity methodId) - { - return _methodIdToMethodDoc.get(methodId); + _currentDocLet = new RestLiDoclet(docInfo, docEnv); + + return true; + } + + public String getDocComment(Element element) { + return _elements.getDocComment(element); + } + + private static class DocInfo { + public TypeElement getClassDoc(String className) { + return classNameToClassDoc.get(className); } - public void setClassDoc(String className, ClassDoc classDoc) - { - _classNameToClassDoc.put(className, classDoc); + public ExecutableElement getMethodDoc(MethodIdentity methodId) { + return methodIdToMethodDoc.get(methodId); } - public void setMethodDoc(MethodIdentity methodId, MethodDoc methodDoc) - { - _methodIdToMethodDoc.put(methodId, methodDoc); + public void setClassDoc(String className, TypeElement classDoc) { + classNameToClassDoc.put(className, classDoc); } - private final Map _classNameToClassDoc = new HashMap<>(); - private final Map _methodIdToMethodDoc = new HashMap<>(); + public void setMethodDoc(MethodIdentity methodId, ExecutableElement methodDoc) { + methodIdToMethodDoc.put(methodId, methodDoc); + } + + private final Map classNameToClassDoc = new HashMap<>(); + private final Map methodIdToMethodDoc = new HashMap<>(); } - private static class MethodIdentity - { - public static MethodIdentity create(Method method) - { + private static class MethodIdentity { + public static MethodIdentity create(Method method) { final List parameterTypeNames = new ArrayList<>(); // type parameters are not included in identity because of differences between reflection and Doclet: // e.g. for Collection: // reflection Type.toString() -> Collection // Doclet Type.toString() -> Collection - for (Class paramClass: method.getParameterTypes()) - { + for (Class paramClass : method.getParameterTypes()) { parameterTypeNames.add(paramClass.getCanonicalName()); } return new MethodIdentity(method.getDeclaringClass().getName() + "." + method.getName(), parameterTypeNames); } - public static MethodIdentity create(MethodDoc method) - { + public static MethodIdentity create(ExecutableElement method) { final List parameterTypeNames = new ArrayList<>(); - for (Parameter param: method.parameters()) - { - Type type = param.type(); - parameterTypeNames.add(type.qualifiedTypeName() + type.dimension()); + for (VariableElement param : method.getParameters()) { + TypeMirror type = param.asType(); + parameterTypeNames.add(type.toString()); } - return new MethodIdentity(method.qualifiedName(), parameterTypeNames); + return new MethodIdentity(method.getEnclosingElement().toString() + "." + method.getSimpleName().toString(), + parameterTypeNames); } - private MethodIdentity(String methodQualifiedName, List parameterTypeNames) - { - _methodQualifiedName = methodQualifiedName; - _parameterTypeNames = parameterTypeNames; + private MethodIdentity(String methodQualifiedName, List parameterTypeNames) { + this.methodQualifiedName = methodQualifiedName; + this.parameterTypeNames = parameterTypeNames; } @Override - public int hashCode() - { - return new HashCodeBuilder(17, 29). - append(_methodQualifiedName). - append(_parameterTypeNames). - toHashCode(); + public int hashCode() { + return new HashCodeBuilder(17, 29) + .append(methodQualifiedName) + .append(parameterTypeNames) + .toHashCode(); } @Override - public boolean equals(Object obj) - { - if (this == obj) - { + public boolean equals(Object obj) { + if (this == obj) { return true; } - if (obj == null) - { + if (obj == null) { return false; } - if (getClass() != obj.getClass()) - { + if (getClass() != obj.getClass()) { return false; } final MethodIdentity other = (MethodIdentity) obj; - return new EqualsBuilder(). - append(_methodQualifiedName, other._methodQualifiedName). - append(_parameterTypeNames, other._parameterTypeNames). - isEquals(); + return new EqualsBuilder() + .append(methodQualifiedName, other.methodQualifiedName) + .append(parameterTypeNames, other.parameterTypeNames) + .isEquals(); } - private final String _methodQualifiedName; - private final List _parameterTypeNames; + private final String methodQualifiedName; + private final List parameterTypeNames; } }