Add an HTTP header parser. The driver for this was an attempt to fix h…

…ttps://issues.apache.org/bugzilla/show_bug.cgi?id=52811

Parsing HTTP headers as per RFC2616 is not always as simple as it first appears. For headers that only use tokens the simple approach will normally be sufficient. However, for the other headers, while simple code meets 99.9% of cases, there are often some edge cases that make things far more complicated.

The purpose of this parser is to let the parser worry about the edge cases. It provides strict parsing of HTTP header values assuming that wrapped header lines have already been unwrapped. (The Tomcat header processing code does the unwrapping.)

The parser currently supports parsing of the following HTTP header values as per RFC 2616:
 - Content-Type

Support for additional headers will be provided as required. A quick scan of the Tomcat code base suggested a couple of places where using this parser may be useful such as  Ranges in the default servlet but there was not - at this point - a compelling case for immediate replacement. The expectation is that as problems are identified in header parsing, the fix will typically extend this parser to support the problematic header and then use the parser rather than custom code.

git-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@1300154 13f79535-47bb-0310-9956-ffa450edef68

.gitignore

@@ -30,6 +30,7 @@ mvn.properties
 .settings
 *.iml
 *.asc
+*.jj
 *.tmp
 maven-ant-tasks-*.jar
 thumbs.db

build.xml

@@ -455,11 +455,12 @@
         <exclude name="res/checkstyle/header-al2.txt"/>
         <!-- Exclude auto-generated files -->
         <exclude name="java/org/apache/el/parser/ELParser*.java" />
-        <exclude name="java/org/apache/el/parser/JJTELParserState.java" />
         <exclude name="java/org/apache/el/parser/Node.java" />
-        <exclude name="java/org/apache/el/parser/ParseException.java" />
-        <exclude name="java/org/apache/el/parser/SimpleCharStream.java" />
-        <exclude name="java/org/apache/el/parser/Token*.java" />
+        <exclude name="java/org/apache/tomcat/util/http/parser/HttpParser*.java" />
+        <exclude name="java/org/apache/**/parser/JJT*ParserState.java" />
+        <exclude name="java/org/apache/**/parser/ParseException.java" />
+        <exclude name="java/org/apache/**/parser/SimpleCharStream.java" />
+        <exclude name="java/org/apache/**/parser/Token*.java" />
         <!-- Exclude these else Gump runs validate on them -->
         <exclude name="**/org/apache/tomcat/dbcp/**"/>
         <exclude name="**/tomcat-deps/**"/>

java/org/apache/coyote/Response.java

@@ -18,10 +18,14 @@
 package org.apache.coyote;
 
 import java.io.IOException;
+import java.io.StringReader;
 import java.util.Locale;
 
 import org.apache.tomcat.util.buf.ByteChunk;
 import org.apache.tomcat.util.http.MimeHeaders;
+import org.apache.tomcat.util.http.parser.AstMediaType;
+import org.apache.tomcat.util.http.parser.HttpParser;
+import org.apache.tomcat.util.http.parser.ParseException;
 
 /**
  * Response object.
@@ -424,71 +428,38 @@ public String getCharacterEncoding() {
      *
      * @param type the content type
      */
-    @SuppressWarnings("deprecation")
     public void setContentType(String type) {
 
-        int semicolonIndex = -1;
-
         if (type == null) {
             this.contentType = null;
             return;
         }
 
-        /*
-         * Remove the charset param (if any) from the Content-Type, and use it
-         * to set the response encoding.
-         * The most recent response encoding setting will be appended to the
-         * response's Content-Type (as its charset param) by getContentType();
-         */
-        boolean hasCharset = false;
-        int len = type.length();
-        int index = type.indexOf(';');
-        while (index != -1) {
-            semicolonIndex = index;
-            index++;
-            // Yes, isSpace() is deprecated but it does exactly what we need
-            while (index < len && Character.isSpace(type.charAt(index))) {
-                index++;
-            }
-            if (index+8 < len
-                    && type.charAt(index) == 'c'
-                    && type.charAt(index+1) == 'h'
-                    && type.charAt(index+2) == 'a'
-                    && type.charAt(index+3) == 'r'
-                    && type.charAt(index+4) == 's'
-                    && type.charAt(index+5) == 'e'
-                    && type.charAt(index+6) == 't'
-                    && type.charAt(index+7) == '=') {
-                hasCharset = true;
-                break;
-            }
-            index = type.indexOf(';', index);
-        }
-
-        if (!hasCharset) {
+        AstMediaType m = null;
+        HttpParser hp = new HttpParser(new StringReader(type));
+        try {
+             m = hp.MediaType();
+        } catch (ParseException e) {
+            // Invalid - Assume no charset and just pass through whatever
+            // the user provided.
             this.contentType = type;
             return;
         }
 
-        this.contentType = type.substring(0, semicolonIndex);
-        String tail = type.substring(index+8);
-        int nextParam = tail.indexOf(';');
-        String charsetValue = null;
-        if (nextParam != -1) {
-            this.contentType += tail.substring(nextParam);
-            charsetValue = tail.substring(0, nextParam);
-        } else {
-            charsetValue = tail;
-        }
+        this.contentType = m.toStringNoCharset();
+
+        String charsetValue = m.getCharset().trim();
 
-        // The charset value may be quoted, but must not contain any quotes.
         if (charsetValue != null && charsetValue.length() > 0) {
-            charsetSet=true;
-            charsetValue = charsetValue.replace('"', ' ');
-            this.characterEncoding = charsetValue.trim();
+            charsetSet = true;
+            this.characterEncoding = charsetValue;
         }
     }
 
+    public void setContentTypeNoCharset(String type) {
+        this.contentType = type;
+    }
+
     public String getContentType() {
 
         String ret = contentType;

java/org/apache/tomcat/util/http/parser/AstAttribute.java

@@ -0,0 +1,36 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.tomcat.util.http.parser;
+
+/**
+ * Represents an attribute as per section 3.6 of RFC 2616. Originally generated
+ * by <a href="http://javacc.java.net/doc/JJTree.html"> JJTree</a>.
+ */
+public class AstAttribute extends SimpleNode {
+    public AstAttribute(int id) {
+        super(id);
+    }
+
+    public AstAttribute(HttpParser p, int id) {
+        super(p, id);
+    }
+
+    @Override
+    public String toString() {
+        return value.toString();
+    }
+}

java/org/apache/tomcat/util/http/parser/AstMediaType.java

@@ -0,0 +1,69 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.tomcat.util.http.parser;
+
+/**
+ * Represents a media-type as per section 3.7 of RFC 2616. Originally generated
+ * by <a href="http://javacc.java.net/doc/JJTree.html"> JJTree</a>.
+ */
+public class AstMediaType extends SimpleNode {
+    public AstMediaType(int id) {
+        super(id);
+    }
+
+    public AstMediaType(HttpParser p, int id) {
+        super(p, id);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(children[0].toString());
+        sb.append("/");
+        sb.append(children[1].toString());
+        for (int i = 2; i < children.length; i++) {
+            sb.append(";");
+            sb.append(children[i].toString());
+        }
+        return sb.toString();
+    }
+
+    public String toStringNoCharset() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(children[0].toString());
+        sb.append("/");
+        sb.append(children[1].toString());
+        for (int i = 2; i < children.length; i++) {
+            AstParameter p = (AstParameter) children[i];
+            if (!"charset".equals(p.children[0].jjtGetValue())) {
+                sb.append(";");
+                sb.append(p.toString());
+            }
+        }
+        return sb.toString();
+    }
+
+    public String getCharset() {
+        for (int i = 2; i < children.length; i++) {
+            AstParameter p = (AstParameter) children[i];
+            if ("charset".equals(p.children[0].jjtGetValue())) {
+                return p.children[1].jjtGetValue().toString();
+            }
+        }
+        return null;
+    }
+}

java/org/apache/tomcat/util/http/parser/AstParameter.java

@@ -0,0 +1,40 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.tomcat.util.http.parser;
+
+/**
+ * Represents a parameter as per section 3.6 of RFC 2616. Originally generated
+ * by <a href="http://javacc.java.net/doc/JJTree.html"> JJTree</a>.
+ */
+public class AstParameter extends SimpleNode {
+    public AstParameter(int id) {
+        super(id);
+    }
+
+    public AstParameter(HttpParser p, int id) {
+        super(p, id);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(children[0].toString());
+        sb.append("=");
+        sb.append(children[1].toString());
+        return sb.toString();
+    }
+}

java/org/apache/tomcat/util/http/parser/AstSubType.java

@@ -0,0 +1,36 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.tomcat.util.http.parser;
+
+/**
+ * Represents a sub-type as per section 3.7 of RFC 2616. Originally generated by
+ * <a href="http://javacc.java.net/doc/JJTree.html"> JJTree</a>.
+ */
+public class AstSubType extends SimpleNode {
+    public AstSubType(int id) {
+        super(id);
+    }
+
+    public AstSubType(HttpParser p, int id) {
+        super(p, id);
+    }
+
+    @Override
+    public String toString() {
+        return value.toString();
+    }
+}

java/org/apache/tomcat/util/http/parser/AstType.java

@@ -0,0 +1,36 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.tomcat.util.http.parser;
+
+/**
+ * Represents a type as per section 3.7 of RFC 2616. Originally generated by <a
+ * href="http://javacc.java.net/doc/JJTree.html"> JJTree</a>.
+ */
+public class AstType extends SimpleNode {
+    public AstType(int id) {
+        super(id);
+    }
+
+    public AstType(HttpParser p, int id) {
+        super(p, id);
+    }
+
+    @Override
+    public String toString() {
+        return value.toString();
+    }
+}

java/org/apache/tomcat/util/http/parser/AstValue.java

@@ -0,0 +1,48 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.tomcat.util.http.parser;
+
+/**
+ * Represents a value as per section 3.6 of RFC 2616. Originally generated by <a
+ * href="http://javacc.java.net/doc/JJTree.html"> JJTree</a>.
+ */
+public class AstValue extends SimpleNode {
+    @Override
+    public Object jjtGetValue() {
+        String s = value.toString();
+        if (s.charAt(0) == '\"') {
+            // Quoted
+            return s.substring(1, s.length() - 1).replaceAll("\\\"", "\"");
+        } else {
+            // Unquoted
+            return s;
+        }
+    }
+
+    public AstValue(int id) {
+        super(id);
+    }
+
+    public AstValue(HttpParser p, int id) {
+        super(p, id);
+    }
+
+    @Override
+    public String toString() {
+        return value.toString();
+    }
+}