001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.io.input;
018
019import java.io.BufferedInputStream;
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.InputStreamReader;
025import java.io.Reader;
026import java.io.StringReader;
027import java.net.HttpURLConnection;
028import java.net.URL;
029import java.net.URLConnection;
030import java.nio.charset.Charset;
031import java.nio.charset.StandardCharsets;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.text.MessageFormat;
035import java.util.Locale;
036import java.util.Objects;
037import java.util.regex.Matcher;
038import java.util.regex.Pattern;
039
040import org.apache.commons.io.ByteOrderMark;
041import org.apache.commons.io.Charsets;
042import org.apache.commons.io.IOUtils;
043import org.apache.commons.io.build.AbstractStreamBuilder;
044import org.apache.commons.io.function.IOConsumer;
045import org.apache.commons.io.output.XmlStreamWriter;
046
047/**
048 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream.
049 * <p>
050 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream.
051 * </p>
052 * <p>
053 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100%
054 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers).
055 * </p>
056 * <p>
057 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors.
058 * </p>
059 * <p>
060 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML
061 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="http://diveintomark.org/archives/2004/02/13/xml-media-types"> Determining
062 * the character encoding of a feed</a>.
063 * </p>
064 * <p>
065 * To build an instance, see {@link Builder}.
066 * </p>
067 * <p>
068 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under Apache License 2.0.
069 * </p>
070 *
071 * @see org.apache.commons.io.output.XmlStreamWriter
072 * @since 2.0
073 */
074public class XmlStreamReader extends Reader {
075
076    /**
077     * Builds a new {@link XmlStreamWriter} instance.
078     *
079     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
080     * <p>
081     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
082     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
083     * </p>
084     * <p>
085     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
086     * </p>
087     * <p>
088     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
089     * </p>
090     * <p>
091     * Else if the XML prolog had a charset encoding that encoding is used.
092     * </p>
093     * <p>
094     * Else if the content type had a charset encoding that encoding is used.
095     * </p>
096     * <p>
097     * Else 'UTF-8' is used.
098     * </p>
099     * <p>
100     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
101     * </p>
102     * <p>
103     * For example:
104     * </p>
105     *
106     * <pre>{@code
107     * XmlStreamReader r = XmlStreamReader.builder()
108     *   .setPath(path)
109     *   .setCharset(StandardCharsets.UTF_8)
110     *   .get();}
111     * </pre>
112     *
113     * @since 2.12.0
114     */
115    public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> {
116
117        private boolean nullCharset = true;
118        private boolean lenient = true;
119        private String httpContentType;
120
121        /**
122         * Constructs a new instance.
123         * <p>
124         * This builder use the aspect InputStream, OpenOption[], httpContentType, lenient, and defaultEncoding.
125         * </p>
126         * <p>
127         * You must provide an origin that can be converted to an InputStream by this builder, otherwise, this call will throw an
128         * {@link UnsupportedOperationException}.
129         * </p>
130         *
131         * @return a new instance.
132         * @throws UnsupportedOperationException if the origin cannot provide an InputStream.
133         * @throws IOException                   thrown if there is a problem reading the stream.
134         * @throws XmlStreamReaderException      thrown if the charset encoding could not be determined according to the specification.
135         * @see #getInputStream()
136         */
137        @SuppressWarnings("resource")
138        @Override
139        public XmlStreamReader get() throws IOException {
140            final String defaultEncoding = nullCharset ? null : getCharset().name();
141            // @formatter:off
142            return httpContentType == null
143                    ? new XmlStreamReader(getInputStream(), lenient, defaultEncoding)
144                    : new XmlStreamReader(getInputStream(), httpContentType, lenient, defaultEncoding);
145            // @formatter:on
146        }
147
148        @Override
149        public Builder setCharset(final Charset charset) {
150            nullCharset = charset == null;
151            return super.setCharset(charset);
152        }
153
154        @Override
155        public Builder setCharset(final String charset) {
156            nullCharset = charset == null;
157            return super.setCharset(Charsets.toCharset(charset, getCharsetDefault()));
158        }
159
160        public Builder setHttpContentType(final String httpContentType) {
161            this.httpContentType = httpContentType;
162            return this;
163        }
164
165        public Builder setLenient(final boolean lenient) {
166            this.lenient = lenient;
167            return this;
168        }
169
170    }
171
172    private static final String UTF_8 = StandardCharsets.UTF_8.name();
173
174    private static final String US_ASCII = StandardCharsets.US_ASCII.name();
175
176    private static final String UTF_16BE = StandardCharsets.UTF_16BE.name();
177
178    private static final String UTF_16LE = StandardCharsets.UTF_16LE.name();
179
180    private static final String UTF_32BE = "UTF-32BE";
181
182    private static final String UTF_32LE = "UTF-32LE";
183
184    private static final String UTF_16 = StandardCharsets.UTF_16.name();
185
186    private static final String UTF_32 = "UTF-32";
187
188    private static final String EBCDIC = "CP1047";
189
190    private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE,
191            ByteOrderMark.UTF_32LE };
192
193    /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */
194    private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D),
195            new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
196            new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
197            new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
198            new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) };
199
200    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?");
201
202    /**
203     * Pattern capturing the encoding of the "xml" processing instruction.
204     * <p>
205     * See also the <a href="https://www.w3.org/TR/2008/REC-xml-20081126/#NT-EncName">XML specification</a>.
206     * </p>
207     */
208    public static final Pattern ENCODING_PATTERN = Pattern.compile(
209    // @formatter:off
210            "^<\\?xml\\s+"
211            + "version\\s*=\\s*(?:(?:\"1\\.[0-9]+\")|(?:'1.[0-9]+'))\\s+"
212            + "encoding\\s*=\\s*((?:\"[A-Za-z]([A-Za-z0-9\\._]|-)*\")|(?:'[A-Za-z]([A-Za-z0-9\\\\._]|-)*'))",
213            Pattern.MULTILINE);
214    // @formatter:on
215
216    private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
217
218    private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
219
220    private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be null";
221
222    private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
223
224    private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME";
225
226    /**
227     * Constructs a new {@link Builder}.
228     *
229     * @return a new {@link Builder}.
230     * @since 2.12.0
231     */
232    public static Builder builder() {
233        return new Builder();
234    }
235
236    /**
237     * Gets the charset parameter value, {@code null} if not present, {@code null} if httpContentType is {@code null}.
238     *
239     * @param httpContentType the HTTP content type
240     * @return The content type encoding (upcased)
241     */
242    static String getContentTypeEncoding(final String httpContentType) {
243        String encoding = null;
244        if (httpContentType != null) {
245            final int i = httpContentType.indexOf(";");
246            if (i > -1) {
247                final String postMime = httpContentType.substring(i + 1);
248                final Matcher m = CHARSET_PATTERN.matcher(postMime);
249                encoding = m.find() ? m.group(1) : null;
250                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
251            }
252        }
253        return encoding;
254    }
255
256    /**
257     * Gets the MIME type or {@code null} if httpContentType is {@code null}.
258     *
259     * @param httpContentType the HTTP content type
260     * @return The mime content type
261     */
262    static String getContentTypeMime(final String httpContentType) {
263        String mime = null;
264        if (httpContentType != null) {
265            final int i = httpContentType.indexOf(";");
266            if (i >= 0) {
267                mime = httpContentType.substring(0, i);
268            } else {
269                mime = httpContentType;
270            }
271            mime = mime.trim();
272        }
273        return mime;
274    }
275
276    /**
277     * Gets the encoding declared in the <?xml encoding=...?>, {@code null} if none.
278     *
279     * @param inputStream InputStream to create the reader from.
280     * @param guessedEnc  guessed encoding
281     * @return the encoding declared in the <?xml encoding=...?>
282     * @throws IOException thrown if there is a problem reading the stream.
283     */
284    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException {
285        String encoding = null;
286        if (guessedEnc != null) {
287            final byte[] bytes = IOUtils.byteArray();
288            inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE);
289            int offset = 0;
290            int max = IOUtils.DEFAULT_BUFFER_SIZE;
291            int c = inputStream.read(bytes, offset, max);
292            int firstGT = -1;
293            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
294            while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) {
295                offset += c;
296                max -= c;
297                c = inputStream.read(bytes, offset, max);
298                xmlProlog = new String(bytes, 0, offset, guessedEnc);
299                firstGT = xmlProlog.indexOf('>');
300            }
301            if (firstGT == -1) {
302                if (c == -1) {
303                    throw new IOException("Unexpected end of XML stream");
304                }
305                throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes");
306            }
307            final int bytesRead = offset;
308            if (bytesRead > 0) {
309                inputStream.reset();
310                final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1)));
311                final StringBuilder prolog = new StringBuilder();
312                IOConsumer.forEach(bReader.lines(), prolog::append);
313                final Matcher m = ENCODING_PATTERN.matcher(prolog);
314                if (m.find()) {
315                    encoding = m.group(1).toUpperCase(Locale.ROOT);
316                    encoding = encoding.substring(1, encoding.length() - 1);
317                }
318            }
319        }
320        return encoding;
321    }
322
323    /**
324     * Tests if the MIME type belongs to the APPLICATION XML family.
325     *
326     * @param mime The mime type
327     * @return true if the mime type belongs to the APPLICATION XML family, otherwise false
328     */
329    static boolean isAppXml(final String mime) {
330        return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity")
331                || mime.startsWith("application/") && mime.endsWith("+xml"));
332    }
333
334    /**
335     * Tests if the MIME type belongs to the TEXT XML family.
336     *
337     * @param mime The mime type
338     * @return true if the mime type belongs to the TEXT XML family, otherwise false
339     */
340    static boolean isTextXml(final String mime) {
341        return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml"));
342    }
343
344    private final Reader reader;
345
346    private final String encoding;
347
348    private final String defaultEncoding;
349
350    /**
351     * Constructs a Reader for a File.
352     * <p>
353     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
354     * </p>
355     * <p>
356     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
357     * </p>
358     *
359     * @param file File to create a Reader from.
360     * @throws NullPointerException if the input is {@code null}.
361     * @throws IOException thrown if there is a problem reading the file.
362     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
363     */
364    @Deprecated
365    public XmlStreamReader(final File file) throws IOException {
366        this(Objects.requireNonNull(file, "file").toPath());
367    }
368
369    /**
370     * Constructs a Reader for a raw InputStream.
371     * <p>
372     * It follows the same logic used for files.
373     * </p>
374     * <p>
375     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
376     * </p>
377     *
378     * @param inputStream InputStream to create a Reader from.
379     * @throws NullPointerException if the input stream is {@code null}.
380     * @throws IOException thrown if there is a problem reading the stream.
381     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
382     */
383    @Deprecated
384    public XmlStreamReader(final InputStream inputStream) throws IOException {
385        this(inputStream, true);
386    }
387
388    /**
389     * Constructs a Reader for a raw InputStream.
390     * <p>
391     * It follows the same logic used for files.
392     * </p>
393     * <p>
394     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
395     * </p>
396     * <p>
397     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
398     * </p>
399     * <p>
400     * Else if the XML prolog had a charset encoding that encoding is used.
401     * </p>
402     * <p>
403     * Else if the content type had a charset encoding that encoding is used.
404     * </p>
405     * <p>
406     * Else 'UTF-8' is used.
407     * </p>
408     * <p>
409     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
410     * </p>
411     *
412     * @param inputStream InputStream to create a Reader from.
413     * @param lenient     indicates if the charset encoding detection should be relaxed.
414     * @throws NullPointerException if the input stream is {@code null}.
415     * @throws IOException              thrown if there is a problem reading the stream.
416     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
417     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
418     */
419    @Deprecated
420    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
421        this(inputStream, lenient, null);
422    }
423
424    /**
425     * Constructs a Reader for a raw InputStream.
426     * <p>
427     * It follows the same logic used for files.
428     * </p>
429     * <p>
430     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
431     * </p>
432     * <p>
433     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
434     * </p>
435     * <p>
436     * Else if the XML prolog had a charset encoding that encoding is used.
437     * </p>
438     * <p>
439     * Else if the content type had a charset encoding that encoding is used.
440     * </p>
441     * <p>
442     * Else 'UTF-8' is used.
443     * </p>
444     * <p>
445     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
446     * </p>
447     *
448     * @param inputStream     InputStream to create a Reader from.
449     * @param lenient         indicates if the charset encoding detection should be relaxed.
450     * @param defaultEncoding The default encoding
451     * @throws NullPointerException if the input stream is {@code null}.
452     * @throws IOException              thrown if there is a problem reading the stream.
453     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
454     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
455     */
456    @Deprecated
457    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
458    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException {
459        Objects.requireNonNull(inputStream, "inputStream");
460        this.defaultEncoding = defaultEncoding;
461        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS);
462        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
463        this.encoding = doRawStream(bom, pis, lenient);
464        this.reader = new InputStreamReader(pis, encoding);
465    }
466
467    /**
468     * Constructs a Reader using an InputStream and the associated content-type header.
469     * <p>
470     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
471     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
472     * </p>
473     * <p>
474     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
475     * </p>
476     *
477     * @param inputStream     InputStream to create the reader from.
478     * @param httpContentType content-type header to use for the resolution of the charset encoding.
479     * @throws NullPointerException if the input stream is {@code null}.
480     * @throws IOException thrown if there is a problem reading the file.
481     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
482     */
483    @Deprecated
484    public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException {
485        this(inputStream, httpContentType, true);
486    }
487
488    /**
489     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
490     * <p>
491     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
492     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
493     * </p>
494     * <p>
495     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
496     * </p>
497     * <p>
498     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
499     * </p>
500     * <p>
501     * Else if the XML prolog had a charset encoding that encoding is used.
502     * </p>
503     * <p>
504     * Else if the content type had a charset encoding that encoding is used.
505     * </p>
506     * <p>
507     * Else 'UTF-8' is used.
508     * </p>
509     * <p>
510     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
511     * </p>
512     *
513     * @param inputStream     InputStream to create the reader from.
514     * @param httpContentType content-type header to use for the resolution of the charset encoding.
515     * @param lenient         indicates if the charset encoding detection should be relaxed.
516     * @throws NullPointerException if the input stream is {@code null}.
517     * @throws IOException              thrown if there is a problem reading the file.
518     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
519     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
520     */
521    @Deprecated
522    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException {
523        this(inputStream, httpContentType, lenient, null);
524    }
525
526    /**
527     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
528     * <p>
529     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
530     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
531     * </p>
532     * <p>
533     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
534     * </p>
535     * <p>
536     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
537     * </p>
538     * <p>
539     * Else if the XML prolog had a charset encoding that encoding is used.
540     * </p>
541     * <p>
542     * Else if the content type had a charset encoding that encoding is used.
543     * </p>
544     * <p>
545     * Else 'UTF-8' is used.
546     * </p>
547     * <p>
548     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
549     * </p>
550     *
551     * @param inputStream     InputStream to create the reader from.
552     * @param httpContentType content-type header to use for the resolution of the charset encoding.
553     * @param lenient         indicates if the charset encoding detection should be relaxed.
554     * @param defaultEncoding The default encoding
555     * @throws NullPointerException if the input stream is {@code null}.
556     * @throws IOException              thrown if there is a problem reading the file.
557     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
558     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
559     */
560    @Deprecated
561    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
562    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding)
563            throws IOException {
564        Objects.requireNonNull(inputStream, "inputStream");
565        this.defaultEncoding = defaultEncoding;
566        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS);
567        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
568        this.encoding = processHttpStream(bom, pis, httpContentType, lenient);
569        this.reader = new InputStreamReader(pis, encoding);
570    }
571
572    /**
573     * Constructs a Reader for a File.
574     * <p>
575     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
576     * </p>
577     * <p>
578     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
579     * </p>
580     *
581     * @param file File to create a Reader from.
582     * @throws NullPointerException if the input is {@code null}.
583     * @throws IOException thrown if there is a problem reading the file.
584     * @since 2.11.0
585     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
586     */
587    @Deprecated
588    @SuppressWarnings("resource") // InputStream is managed through another reader in this instance.
589    public XmlStreamReader(final Path file) throws IOException {
590        this(Files.newInputStream(Objects.requireNonNull(file, "file")));
591    }
592
593    /**
594     * Constructs a Reader using the InputStream of a URL.
595     * <p>
596     * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files.
597     * </p>
598     * <p>
599     * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type.
600     * </p>
601     * <p>
602     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
603     * </p>
604     *
605     * @param url URL to create a Reader from.
606     * @throws NullPointerException if the input is {@code null}.
607     * @throws IOException thrown if there is a problem reading the stream of the URL.
608     */
609    public XmlStreamReader(final URL url) throws IOException {
610        this(Objects.requireNonNull(url, "url").openConnection(), null);
611    }
612
613    /**
614     * Constructs a Reader using the InputStream of a URLConnection.
615     * <p>
616     * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files.
617     * </p>
618     * <p>
619     * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with
620     * content-type.
621     * </p>
622     * <p>
623     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
624     * </p>
625     *
626     * @param urlConnection   URLConnection to create a Reader from.
627     * @param defaultEncoding The default encoding
628     * @throws NullPointerException if the input is {@code null}.
629     * @throws IOException thrown if there is a problem reading the stream of the URLConnection.
630     */
631    public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException {
632        Objects.requireNonNull(urlConnection, "urlConnection");
633        this.defaultEncoding = defaultEncoding;
634        final boolean lenient = true;
635        final String contentType = urlConnection.getContentType();
636        final InputStream inputStream = urlConnection.getInputStream();
637        @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance
638        // @formatter:off
639        final BOMInputStream bomInput = BOMInputStream.builder()
640            .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE))
641            .setInclude(false)
642            .setByteOrderMarks(BOMS)
643            .get();
644        @SuppressWarnings("resource")
645        final BOMInputStream piInput = BOMInputStream.builder()
646            .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE))
647            .setInclude(true)
648            .setByteOrderMarks(XML_GUESS_BYTES)
649            .get();
650        // @formatter:on
651        if (urlConnection instanceof HttpURLConnection || contentType != null) {
652            this.encoding = processHttpStream(bomInput, piInput, contentType, lenient);
653        } else {
654            this.encoding = doRawStream(bomInput, piInput, lenient);
655        }
656        this.reader = new InputStreamReader(piInput, encoding);
657    }
658
659    /**
660     * Calculates the HTTP encoding.
661     *
662     * @param httpContentType The HTTP content type
663     * @param bomEnc          BOM encoding
664     * @param xmlGuessEnc     XML Guess encoding
665     * @param xmlEnc          XML encoding
666     * @param lenient         indicates if the charset encoding detection should be relaxed.
667     * @return the HTTP encoding
668     * @throws IOException thrown if there is a problem reading the stream.
669     */
670    String calculateHttpEncoding(final String httpContentType, final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient)
671            throws IOException {
672
673        // Lenient and has XML encoding
674        if (lenient && xmlEnc != null) {
675            return xmlEnc;
676        }
677
678        // Determine mime/encoding content types from HTTP Content Type
679        final String cTMime = getContentTypeMime(httpContentType);
680        final String cTEnc = getContentTypeEncoding(httpContentType);
681        final boolean appXml = isAppXml(cTMime);
682        final boolean textXml = isTextXml(cTMime);
683
684        // Mime type NOT "application/xml" or "text/xml"
685        if (!appXml && !textXml) {
686            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
687            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
688        }
689
690        // No content type encoding
691        if (cTEnc == null) {
692            if (appXml) {
693                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
694            }
695            return defaultEncoding == null ? US_ASCII : defaultEncoding;
696        }
697
698        // UTF-16BE or UTF-16LE content type encoding
699        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
700            if (bomEnc != null) {
701                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
702                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
703            }
704            return cTEnc;
705        }
706
707        // UTF-16 content type encoding
708        if (cTEnc.equals(UTF_16)) {
709            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
710                return bomEnc;
711            }
712            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
713            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
714        }
715
716        // UTF-32BE or UTF-132E content type encoding
717        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
718            if (bomEnc != null) {
719                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
720                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
721            }
722            return cTEnc;
723        }
724
725        // UTF-32 content type encoding
726        if (cTEnc.equals(UTF_32)) {
727            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
728                return bomEnc;
729            }
730            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
731            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
732        }
733
734        return cTEnc;
735    }
736
737    /**
738     * Calculate the raw encoding.
739     *
740     * @param bomEnc      BOM encoding
741     * @param xmlGuessEnc XML Guess encoding
742     * @param xmlEnc      XML encoding
743     * @return the raw encoding
744     * @throws IOException thrown if there is a problem reading the stream.
745     */
746    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException {
747
748        // BOM is Null
749        if (bomEnc == null) {
750            if (xmlGuessEnc == null || xmlEnc == null) {
751                return defaultEncoding == null ? UTF_8 : defaultEncoding;
752            }
753            if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
754                return xmlGuessEnc;
755            }
756            return xmlEnc;
757        }
758
759        // BOM is UTF-8
760        if (bomEnc.equals(UTF_8)) {
761            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
762                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
763                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
764            }
765            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
766                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
767                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
768            }
769            return bomEnc;
770        }
771
772        // BOM is UTF-16BE or UTF-16LE
773        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
774            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
775                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
776                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
777            }
778            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
779                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
780                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
781            }
782            return bomEnc;
783        }
784
785        // BOM is UTF-32BE or UTF-32LE
786        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
787            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
788                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
789                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
790            }
791            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
792                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
793                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
794            }
795            return bomEnc;
796        }
797
798        // BOM is something else
799        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
800        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
801    }
802
803    /**
804     * Closes the XmlStreamReader stream.
805     *
806     * @throws IOException thrown if there was a problem closing the stream.
807     */
808    @Override
809    public void close() throws IOException {
810        reader.close();
811    }
812
813    /**
814     * Does lenient detection.
815     *
816     * @param httpContentType content-type header to use for the resolution of the charset encoding.
817     * @param ex              The thrown exception
818     * @return the encoding
819     * @throws IOException thrown if there is a problem reading the stream.
820     */
821    private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException {
822        if (httpContentType != null && httpContentType.startsWith("text/html")) {
823            httpContentType = httpContentType.substring("text/html".length());
824            httpContentType = "text/xml" + httpContentType;
825            try {
826                return calculateHttpEncoding(httpContentType, ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true);
827            } catch (final XmlStreamReaderException ex2) {
828                ex = ex2;
829            }
830        }
831        String encoding = ex.getXmlEncoding();
832        if (encoding == null) {
833            encoding = ex.getContentTypeEncoding();
834        }
835        if (encoding == null) {
836            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
837        }
838        return encoding;
839    }
840
841    /**
842     * Process the raw stream.
843     *
844     * @param bom     BOMInputStream to detect byte order marks
845     * @param pis     BOMInputStream to guess XML encoding
846     * @param lenient indicates if the charset encoding detection should be relaxed.
847     * @return the encoding to be used
848     * @throws IOException thrown if there is a problem reading the stream.
849     */
850    private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient) throws IOException {
851        final String bomEnc = bom.getBOMCharsetName();
852        final String xmlGuessEnc = pis.getBOMCharsetName();
853        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
854        try {
855            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
856        } catch (final XmlStreamReaderException ex) {
857            if (lenient) {
858                return doLenientDetection(null, ex);
859            }
860            throw ex;
861        }
862    }
863
864    /**
865     * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate.
866     * <p>
867     * If it is {@code null} the content-type based rules are used.
868     * </p>
869     *
870     * @return the default encoding to use.
871     */
872    public String getDefaultEncoding() {
873        return defaultEncoding;
874    }
875
876    /**
877     * Gets the charset encoding of the XmlStreamReader.
878     *
879     * @return charset encoding.
880     */
881    public String getEncoding() {
882        return encoding;
883    }
884
885    /**
886     * Processes an HTTP stream.
887     *
888     * @param bomInput        BOMInputStream to detect byte order marks
889     * @param piInput         BOMInputStream to guess XML encoding
890     * @param httpContentType The HTTP content type
891     * @param lenient         indicates if the charset encoding detection should be relaxed.
892     * @return the encoding to be used
893     * @throws IOException thrown if there is a problem reading the stream.
894     */
895    private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final String httpContentType, final boolean lenient)
896            throws IOException {
897        final String bomEnc = bomInput.getBOMCharsetName();
898        final String xmlGuessEnc = piInput.getBOMCharsetName();
899        final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc);
900        try {
901            return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient);
902        } catch (final XmlStreamReaderException ex) {
903            if (lenient) {
904                return doLenientDetection(httpContentType, ex);
905            }
906            throw ex;
907        }
908    }
909
910    /**
911     * Reads the underlying reader's {@code read(char[], int, int)} method.
912     *
913     * @param buf    the buffer to read the characters into
914     * @param offset The start offset
915     * @param len    The number of bytes to read
916     * @return the number of characters read or -1 if the end of stream
917     * @throws IOException if an I/O error occurs.
918     */
919    @Override
920    public int read(final char[] buf, final int offset, final int len) throws IOException {
921        return reader.read(buf, offset, len);
922    }
923
924}