Skip to content

Commit

Permalink
HADOOP-19231. Add JacksonUtil to manage Jackson classes (#6953)
Browse files Browse the repository at this point in the history
New class org.apache.hadoop.util.JacksonUtil centralizes construction of
Jackson ObjectMappers and JsonFactories.

Contributed by PJ Fanning
  • Loading branch information
pjfanning authored Aug 15, 2024
1 parent 55a5769 commit fa9bb0d
Show file tree
Hide file tree
Showing 71 changed files with 392 additions and 296 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import com.ctc.wstx.io.StreamBootstrapper;
import com.ctc.wstx.io.SystemId;
import com.ctc.wstx.stax.WstxInputFactory;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;

import java.io.BufferedInputStream;
Expand Down Expand Up @@ -101,6 +100,7 @@
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
import org.apache.hadoop.util.ConfigurationHelper;
import org.apache.hadoop.util.JacksonUtil;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
Expand Down Expand Up @@ -3792,8 +3792,7 @@ public static void dumpConfiguration(Configuration config,
throw new IllegalArgumentException("Property " +
propertyName + " not found");
} else {
JsonFactory dumpFactory = new JsonFactory();
JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
JsonGenerator dumpGenerator = JacksonUtil.getSharedWriter().createGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
appendJSONProperty(dumpGenerator, config, propertyName,
Expand Down Expand Up @@ -3831,8 +3830,7 @@ public static void dumpConfiguration(Configuration config,
*/
public static void dumpConfiguration(Configuration config,
Writer out) throws IOException {
JsonFactory dumpFactory = new JsonFactory();
JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
JsonGenerator dumpGenerator = JacksonUtil.getSharedWriter().createGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.hadoop.util.JacksonUtil;
import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.KMSUtil;
import org.apache.http.client.utils.URIBuilder;
Expand Down Expand Up @@ -78,7 +79,6 @@
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
Expand Down Expand Up @@ -592,11 +592,10 @@ private <T> T call(HttpURLConnection conn, Object jsonOutput,
&& conn.getContentType().trim().toLowerCase()
.startsWith(APPLICATION_JSON_MIME)
&& klass != null) {
ObjectMapper mapper = new ObjectMapper();
InputStream is = null;
try {
is = conn.getInputStream();
ret = mapper.readValue(is, klass);
ret = JacksonUtil.getSharedReader().readValue(is, klass);
} finally {
IOUtils.closeStream(is);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@

import javax.management.ObjectName;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;

import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.JacksonUtil;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AtomicDoubleArray;
import org.apache.commons.lang3.exception.ExceptionUtils;
Expand Down Expand Up @@ -146,7 +146,7 @@ public class DecayRpcScheduler implements RpcScheduler,
public static final Logger LOG =
LoggerFactory.getLogger(DecayRpcScheduler.class);

private static final ObjectWriter WRITER = new ObjectMapper().writer();
private static final ObjectWriter WRITER = JacksonUtil.getSharedWriter();

// Track the decayed and raw (no decay) number of calls for each schedulable
// identity from all previous decay windows: idx 0 for decayed call cost and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.JacksonUtil;
import org.apache.hadoop.util.ProtoUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
Expand All @@ -130,7 +131,6 @@
import org.apache.hadoop.tracing.TraceScope;
import org.apache.hadoop.tracing.Tracer;
import org.apache.hadoop.tracing.TraceUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.classification.VisibleForTesting;

import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
Expand Down Expand Up @@ -3843,9 +3843,8 @@ public int getNumOpenConnections() {
* @return Get the NumOpenConnections/User.
*/
public String getNumOpenConnectionsPerUser() {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper
return JacksonUtil.getSharedWriter()
.writeValueAsString(connectionManager.getUserToConnectionsMap());
} catch (IOException ignored) {
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.util.JacksonUtil;

/*
* This servlet is based off of the JMXProxyServlet from Tomcat 7.0.14. It has
Expand Down Expand Up @@ -134,19 +134,13 @@ public class JMXJsonServlet extends HttpServlet {
*/
protected transient MBeanServer mBeanServer;

/**
* Json Factory to create Json generators for write objects in json format
*/
protected transient JsonFactory jsonFactory;

/**
* Initialize this servlet.
*/
@Override
public void init() throws ServletException {
// Retrieve the MBean server
mBeanServer = ManagementFactory.getPlatformMBeanServer();
jsonFactory = new JsonFactory();
}

protected boolean isInstrumentationAccessAllowed(HttpServletRequest request,
Expand Down Expand Up @@ -187,7 +181,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) {
response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, "GET");
response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");

jg = jsonFactory.createGenerator(writer);
jg = JacksonUtil.getSharedWriter().createGenerator(writer);
jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
jg.useDefaultPrettyPrinter();
jg.writeStartObject();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.JacksonUtil;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -46,8 +46,7 @@ public class MetricsJsonBuilder extends MetricsRecordBuilder {
private final MetricsCollector parent;
private Map<String, Object> innerMetrics = new LinkedHashMap<>();

private static final ObjectWriter WRITER =
new ObjectMapper().writer();
private static final ObjectWriter WRITER = JacksonUtil.getSharedWriter();

/**
* Build an instance.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.hadoop.util.JacksonUtil;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -165,7 +166,7 @@ public void initTokenManager(Properties config) {
@VisibleForTesting
public void initJsonFactory(Properties config) {
boolean hasFeature = false;
JsonFactory tmpJsonFactory = new JsonFactory();
JsonFactory tmpJsonFactory = JacksonUtil.createBasicJsonFactory();

for (Map.Entry entry : config.entrySet()) {
String key = (String)entry.getKey();
Expand Down Expand Up @@ -335,7 +336,7 @@ public boolean managementOperation(AuthenticationToken token,
if (map != null) {
response.setContentType(MediaType.APPLICATION_JSON);
Writer writer = response.getWriter();
ObjectMapper jsonMapper = new ObjectMapper(jsonFactory);
ObjectMapper jsonMapper = JacksonUtil.createObjectMapper(jsonFactory);
jsonMapper.writeValue(writer, map);
writer.write(ENTER);
writer.flush();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.json.JsonMapper;

import org.apache.hadoop.classification.InterfaceAudience.Private;

/**
* Utility for sharing code related to Jackson usage in Hadoop.
*/
@Private
public final class JacksonUtil {

private static final ObjectMapper SHARED_BASIC_OBJECT_MAPPER = createBasicObjectMapper();
private static final ObjectReader SHARED_BASIC_OBJECT_READER =
SHARED_BASIC_OBJECT_MAPPER.reader();
private static final ObjectWriter SHARED_BASIC_OBJECT_WRITER =
SHARED_BASIC_OBJECT_MAPPER.writer();
private static final ObjectWriter SHARED_BASIC_OBJECT_WRITER_PRETTY =
SHARED_BASIC_OBJECT_MAPPER.writerWithDefaultPrettyPrinter();

/**
* Creates a new {@link JsonFactory} instance with basic configuration.
*
* @return an {@link JsonFactory} with basic configuration
*/
public static JsonFactory createBasicJsonFactory() {
// deliberately return a new instance instead of sharing one because we can't trust
// that users won't modify this instance
return new JsonFactory();
}

/**
* Creates a new {@link ObjectMapper} instance with basic configuration.
*
* @return an {@link ObjectMapper} with basic configuration
*/
public static ObjectMapper createBasicObjectMapper() {
// deliberately return a new instance instead of sharing one because we can't trust
// that users won't modify this instance
return JsonMapper.builder(createBasicJsonFactory()).build();
}

/**
* Creates a new {@link ObjectMapper} instance based on the configuration
* in the input {@link JsonFactory}.
*
* @param jsonFactory a pre-configured {@link JsonFactory}
* @return an {@link ObjectMapper} with configuration set by the input {@link JsonFactory}.
*/
public static ObjectMapper createObjectMapper(final JsonFactory jsonFactory) {
return JsonMapper.builder(jsonFactory).build();
}

/**
* Returns a shared {@link ObjectReader} instance with basic configuration.
*
* @return a shared {@link ObjectReader} instance with basic configuration
*/
public static ObjectReader getSharedReader() {
return SHARED_BASIC_OBJECT_READER;
}

/**
* Returns an {@link ObjectReader} for the given type instance with basic configuration.
*
* @param type the class that the reader has to support
* @return an {@link ObjectReader} instance with basic configuration
*/
public static ObjectReader createBasicReaderFor(Class<?> type) {
return SHARED_BASIC_OBJECT_MAPPER.readerFor(type);
}

/**
* Returns a shared {@link ObjectWriter} instance with basic configuration.
*
* @return a shared {@link ObjectWriter} instance with basic configuration
*/
public static ObjectWriter getSharedWriter() {
return SHARED_BASIC_OBJECT_WRITER;
}

/**
* Returns a shared {@link ObjectWriter} instance with pretty print and basic configuration.
*
* @return a shared {@link ObjectWriter} instance with pretty print and basic configuration
*/
public static ObjectWriter getSharedWriterWithPrettyPrint() {
return SHARED_BASIC_OBJECT_WRITER_PRETTY;
}

/**
* Returns an {@link ObjectWriter} for the given type instance with basic configuration.
*
* @param type the class that the writer has to support
* @return an {@link ObjectWriter} instance with basic configuration
*/
public static ObjectWriter createBasicWriterFor(Class<?> type) {
return SHARED_BASIC_OBJECT_MAPPER.writerFor(type);
}

private JacksonUtil() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,8 @@ public class JsonSerialization<T> {
private final Class<T> classType;
private final ObjectMapper mapper;

private static final ObjectWriter WRITER =
new ObjectMapper().writerWithDefaultPrettyPrinter();

private static final ObjectReader MAP_READER =
new ObjectMapper().readerFor(Map.class);
private static final ObjectWriter WRITER = JacksonUtil.getSharedWriterWithPrettyPrint();
private static final ObjectReader MAP_READER = JacksonUtil.createBasicReaderFor(Map.class);

/**
* @return an ObjectWriter which pretty-prints its output
Expand All @@ -106,7 +103,7 @@ public JsonSerialization(Class<T> classType,
boolean failOnUnknownProperties, boolean pretty) {
Preconditions.checkArgument(classType != null, "null classType");
this.classType = classType;
this.mapper = new ObjectMapper();
this.mapper = JacksonUtil.createBasicObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
failOnUnknownProperties);
mapper.configure(SerializationFeature.INDENT_OUTPUT, pretty);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@
*/
package org.apache.hadoop.crypto.key.kms.server;

import com.fasterxml.jackson.databind.ObjectMapper;

import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.util.JacksonUtil;

import javax.ws.rs.Consumes;
import javax.ws.rs.WebApplicationException;
Expand All @@ -38,7 +37,6 @@
@Consumes(MediaType.APPLICATION_JSON)
@InterfaceAudience.Private
public class KMSJSONReader implements MessageBodyReader<Object> {
private static final ObjectMapper MAPPER = new ObjectMapper();

@Override
public boolean isReadable(Class<?> type, Type genericType,
Expand All @@ -52,6 +50,6 @@ public Object readFrom(Class<Object> type, Type genericType,
Annotation[] annotations, MediaType mediaType,
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
throws IOException, WebApplicationException {
return MAPPER.readValue(entityStream, type);
return JacksonUtil.getSharedReader().readValue(entityStream, type);
}
}
Loading

0 comments on commit fa9bb0d

Please sign in to comment.