From 439ffe2e8ae113f6c2731e604979ddad6ba705e4 Mon Sep 17 00:00:00 2001 From: Arjen Poutsma Date: Thu, 20 Feb 2020 10:48:41 +0100 Subject: [PATCH] Convert non-UTF-8 JSON Jackson's asynchronous parser does not support any encoding except UTF-8 (or ASCII). This commit converts non-UTF-8/ASCII encoded JSON to UTF-8. Closes gh-24489 --- .../codec/json/AbstractJackson2Decoder.java | 23 +++++++++- .../http/codec/json/Jackson2JsonDecoder.java | 45 ++++++++++++++++++- .../codec/json/Jackson2JsonDecoderTests.java | 37 ++++++++++++++- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java b/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java index 98e22e3a05..c77e49263b 100644 --- a/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java +++ b/spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java @@ -120,11 +120,29 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple forceUseOfBigDecimal = true; } - Flux tokens = Jackson2Tokenizer.tokenize(Flux.from(input), this.jsonFactory, getObjectMapper(), + Flux processed = processInput(input, elementType, mimeType, hints); + Flux tokens = Jackson2Tokenizer.tokenize(processed, this.jsonFactory, getObjectMapper(), true, forceUseOfBigDecimal, getMaxInMemorySize()); return decodeInternal(tokens, elementType, mimeType, hints); } + /** + * Process the input publisher into a flux. Default implementation returns + * {@link Flux#from(Publisher)}, but subclasses can choose to to customize + * this behaviour. + * @param input the {@code DataBuffer} input stream to process + * @param elementType the expected type of elements in the output stream + * @param mimeType the MIME type associated with the input stream (optional) + * @param hints additional information about how to do encode + * @return the processed flux + * @since 5.1.14 + */ + protected Flux processInput(Publisher input, ResolvableType elementType, + @Nullable MimeType mimeType, @Nullable Map hints) { + + return Flux.from(input); + } + @Override public Mono decodeToMono(Publisher input, ResolvableType elementType, @Nullable MimeType mimeType, @Nullable Map hints) { @@ -134,7 +152,8 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple forceUseOfBigDecimal = true; } - Flux tokens = Jackson2Tokenizer.tokenize(Flux.from(input), this.jsonFactory, getObjectMapper(), + Flux processed = processInput(input, elementType, mimeType, hints); + Flux tokens = Jackson2Tokenizer.tokenize(processed, this.jsonFactory, getObjectMapper(), false, forceUseOfBigDecimal, getMaxInMemorySize()); return decodeInternal(tokens, elementType, mimeType, hints).singleOrEmpty(); } diff --git a/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2JsonDecoder.java b/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2JsonDecoder.java index b9372ff583..861fa05be2 100644 --- a/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2JsonDecoder.java +++ b/spring-web/src/main/java/org/springframework/http/codec/json/Jackson2JsonDecoder.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2018 the original author or authors. + * Copyright 2002-2020 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,24 @@ package org.springframework.http.codec.json; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; + import com.fasterxml.jackson.databind.ObjectMapper; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import org.springframework.core.ResolvableType; +import org.springframework.core.codec.StringDecoder; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferFactory; +import org.springframework.core.io.buffer.DefaultDataBufferFactory; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; +import org.springframework.lang.Nullable; import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; /** * Decode a byte stream into JSON and convert to Object's with Jackson 2.9, @@ -32,6 +46,11 @@ import org.springframework.util.MimeType; */ public class Jackson2JsonDecoder extends AbstractJackson2Decoder { + private static final StringDecoder STRING_DECODER = StringDecoder.textPlainOnly(Arrays.asList(",", "\n"), false); + + private static final ResolvableType STRING_TYPE = ResolvableType.forClass(String.class); + + public Jackson2JsonDecoder() { super(Jackson2ObjectMapperBuilder.json().build()); } @@ -40,4 +59,28 @@ public class Jackson2JsonDecoder extends AbstractJackson2Decoder { super(mapper, mimeTypes); } + @Override + protected Flux processInput(Publisher input, ResolvableType elementType, + @Nullable MimeType mimeType, @Nullable Map hints) { + + Flux flux = Flux.from(input); + if (mimeType == null) { + return flux; + } + + // Jackson asynchronous parser only supports UTF-8 + Charset charset = mimeType.getCharset(); + if (charset == null || StandardCharsets.UTF_8.equals(charset) || StandardCharsets.US_ASCII.equals(charset)) { + return flux; + } + + // Potentially, the memory consumption of this conversion could be improved by using CharBuffers instead + // of allocating Strings, but that would require refactoring the buffer tokenization code from StringDecoder + + MimeType textMimeType = new MimeType(MimeTypeUtils.TEXT_PLAIN, charset); + Flux decoded = STRING_DECODER.decode(input, STRING_TYPE, textMimeType, null); + DataBufferFactory factory = new DefaultDataBufferFactory(); + return decoded.map(s -> factory.wrap(s.getBytes(StandardCharsets.UTF_8))); + } + } diff --git a/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2JsonDecoderTests.java b/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2JsonDecoderTests.java index e3b158ee73..5252083c0b 100644 --- a/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2JsonDecoderTests.java +++ b/spring-web/src/test/java/org/springframework/http/codec/json/Jackson2JsonDecoderTests.java @@ -18,6 +18,7 @@ package org.springframework.http.codec.json; import java.io.IOException; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; @@ -34,6 +35,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; +import org.springframework.core.ParameterizedTypeReference; import org.springframework.core.ResolvableType; import org.springframework.core.codec.AbstractDecoderTestCase; import org.springframework.core.codec.CodecException; @@ -218,9 +220,42 @@ public class Jackson2JsonDecoderTests extends AbstractDecoderTestCase input = stringBuffer("{\"foo\":\"bar\"}", StandardCharsets.UTF_16); + + testDecode(input, ResolvableType.forType(new ParameterizedTypeReference>() {}), + step -> step.assertNext(o -> { + Map map = (Map) o; + assertEquals("bar", map.get("foo")); + }) + .verifyComplete(), + MediaType.parseMediaType("application/json; charset=utf-16"), + null); + } + + @Test + public void decodeMonoNonUtf8Encoding() { + Mono input = stringBuffer("{\"foo\":\"bar\"}", StandardCharsets.UTF_16); + + testDecodeToMono(input, ResolvableType.forType(new ParameterizedTypeReference>() { + }), + step -> step.assertNext(o -> { + Map map = (Map) o; + assertEquals("bar", map.get("foo")); + }) + .verifyComplete(), + MediaType.parseMediaType("application/json; charset=utf-16"), + null); + } + private Mono stringBuffer(String value) { + return stringBuffer(value, StandardCharsets.UTF_8); + } + + private Mono stringBuffer(String value, Charset charset) { return Mono.defer(() -> { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + byte[] bytes = value.getBytes(charset); DataBuffer buffer = this.bufferFactory.allocateBuffer(bytes.length); buffer.write(bytes); return Mono.just(buffer);