package org.jsoup.integration;
import org.jsoup.Jsoup;
import org.jsoup.helper.DataUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.ParseErrorList;
import org.jsoup.parser.Parser;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.Test;
import java.io.*;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.zip.GZIPInputStream;
import static org.junit.jupiter.api.Assertions.*;
/**
* Integration test: parses from real-world example HTML.
*
* @author Jonathan Hedley, jonathan@hedley.net
*/
public class ParseTest {
@Test
public void testHtml5Charset() throws IOException {
// test that works
File in = getFile("/htmltests/meta-charset-1.html");
Document doc = Jsoup.parse(in, null, "http://example.com/"); //gb2312, has html5
assertEquals("新", doc.text());
assertEquals("GB2312", doc.outputSettings().charset().displayName());
// double check, no charset, falls back to utf8 which is incorrect
in = getFile("/htmltests/meta-charset-2.html"); //
doc = Jsoup.parse(in, null, "http://example.com"); // gb2312, no charset
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
assertNotEquals("新", doc.text());
// confirm fallback to utf8
in = getFile("/htmltests/meta-charset-3.html");
doc = Jsoup.parse(in, null, "http://example.com/"); // utf8, no charset
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
assertEquals("新", doc.text());
}
@Test
public void testBrokenHtml5CharsetWithASingleDoubleQuote() throws IOException {
InputStream in = inputStreamFrom("\n" +
"\n" +
"\n" +
"");
Document doc = Jsoup.parse(in, null, "http://example.com/");
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
}
@Test
public void testLowercaseUtf8Charset() throws IOException {
File in = getFile("/htmltests/lowercase-charset-test.html");
Document doc = Jsoup.parse(in, null);
Element form = doc.select("#form").first();
assertEquals(2, form.children().size());
assertEquals("UTF-8", doc.outputSettings().charset().name());
}
@Test
public void testXwiki() throws IOException {
// https://github.com/jhy/jsoup/issues/1324
// this tests that when in CharacterReader we hit a buffer while marked, we preserve the mark when buffered up and can rewind
File in = getFile("/htmltests/xwiki-1324.html.gz");
Document doc = Jsoup.parse(in, null, "https://localhost/");
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
// was getting busted at =userdirectory, because it hit the bufferup point but the mark was then lost. so
// updated to preserve the mark.
String wantHtml = "User Directory";
assertEquals(wantHtml, doc.select("[data-id=userdirectory]").outerHtml());
}
@Test
public void testXwikiExpanded() throws IOException {
// https://github.com/jhy/jsoup/issues/1324
// this tests that if there is a huge illegal character reference, we can get through a buffer and rewind, and still catch that it's an invalid refence,
// and the parse tree is correct.
File in = getFile("/htmltests/xwiki-edit.html.gz");
Parser parser = Parser.htmlParser();
Document doc = Jsoup.parse(new GZIPInputStream(new FileInputStream(in)), "UTF-8", "https://localhost/", parser.setTrackErrors(100));
ParseErrorList errors = parser.getErrors();
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
assertEquals(0, errors.size()); // not an invalid reference because did not look legit
// was getting busted at =userdirectory, because it hit the bufferup point but the mark was then lost. so
// updated to preserve the mark.
String wantHtml = "User Directory";
assertEquals(wantHtml, doc.select("[data-id=userdirectory]").outerHtml());
}
@Test public void testFileParseNoCharsetMethod() throws IOException {
File in = getFile("/htmltests/xwiki-1324.html.gz");
Document doc = Jsoup.parse(in);
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
}
public static File getFile(String resourceName) {
try {
URL resource = ParseTest.class.getResource(resourceName);
return resource != null ? new File(resource.toURI()) : new File("/404");
} catch (URISyntaxException e) {
throw new IllegalStateException(e);
}
}
public static InputStream inputStreamFrom(String s) {
return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
}
public static String getFileAsString(File file) throws IOException {
byte[] bytes;
if (file.getName().endsWith(".gz")) {
InputStream stream = new GZIPInputStream(new FileInputStream(file));
ByteBuffer byteBuffer = DataUtil.readToByteBuffer(stream, 0);
bytes = byteBuffer.array();
} else {
bytes = Files.readAllBytes(file.toPath());
}
return new String(bytes);
}
}