Current File : //proc/309157/root/home/waritko/yacy/source/net/yacy/document/parser/GenericXMLParser.java
// GenericXMLParser.java
// ---------------------------
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.document.parser;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.naming.SizeLimitExceededException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.commons.io.input.XmlStreamReader;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.AnchorURL;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.StreamLimitException;
import net.yacy.cora.util.StrictLimitInputStream;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
import net.yacy.document.VocabularyScraper;
import net.yacy.document.parser.xml.GenericXMLContentHandler;
import net.yacy.kelondro.io.CharBuffer;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
/**
* A generic XML parser without knowledge of the specific XML vocabulary.
* @author luccioman
*
*/
public class GenericXMLParser extends AbstractParser implements Parser {
/** SAX parser instance local to each thread */
private static final ThreadLocal<SAXParser> tlSax = new ThreadLocal<SAXParser>();
/**
* @return a SAXParser instance for the current thread
* @throws SAXException when an error prevented parser creation
*/
private static SAXParser getParser() throws SAXException {
SAXParser parser = tlSax.get();
if (parser == null) {
try {
parser = SAXParserFactory.newInstance().newSAXParser();
} catch (final ParserConfigurationException e) {
throw new SAXException(e.getMessage(), e);
}
tlSax.set(parser);
}
return parser;
}
public GenericXMLParser() {
super("XML Parser");
this.SUPPORTED_EXTENSIONS.add("xml");
this.SUPPORTED_MIME_TYPES.add("application/xml");
this.SUPPORTED_MIME_TYPES.add("text/xml");
}
@Override
public Document[] parse(
final DigestURL location,
final String mimeType,
final String charset,
final VocabularyScraper scraper,
final int timezoneOffset,
final InputStream source)
throws Failure {
/* Limit the size of the in-memory buffer to at most 25% of the available memory :
* because some room is needed, and before being garbage collected the buffer will be converted to a String, then to a byte array.
* Eventual stricter limits should be handled by the caller (see for example crawler.[protocol].maxFileSize configuration setting). */
final long availableMemory = MemoryControl.available();
final long maxBytes = (long)(availableMemory * 0.25);
final int maxChars;
if((maxBytes / Character.BYTES) > Integer.MAX_VALUE) {
maxChars = Integer.MAX_VALUE;
} else {
maxChars = ((int)maxBytes) / Character.BYTES;
}
try (/* Automatically closed by this try-with-resources statement*/ CharBuffer writer = new CharBuffer(maxChars);){
/* Use commons-io XmlStreamReader advanced rules to help with charset detection when source contains no BOM or XML declaration
* (detection algorithm notably also include ContentType transmitted by HTTP headers, here eventually present as mimeType and charset parameters), */
final XmlStreamReader reader = new XmlStreamReader(source, mimeType, true, charset);
final InputSource saxSource = new InputSource(reader);
final String detectedCharset = reader.getEncoding();
final List<AnchorURL> detectedURLs = new ArrayList<>();
final GenericXMLContentHandler saxHandler = new GenericXMLContentHandler(writer, detectedURLs);
final SAXParser saxParser = getParser();
saxParser.parse(saxSource, saxHandler);
if (writer.isOverflow()) {
throw new Parser.Failure("Not enough Memory available for generic the XML parser : "
+ Formatter.bytesToString(availableMemory), location);
}
/* create the parsed document */
Document[] docs = null;
final byte[] contentBytes = UTF8.getBytes(writer.toString());
docs = new Document[] { new Document(location, mimeType, detectedCharset, this, null, null, null, null, "",
null, null, 0.0d, 0.0d, contentBytes, detectedURLs, null, null, false, new Date()) };
return docs;
} catch(Parser.Failure e) {
throw e;
} catch (final Exception e) {
throw new Parser.Failure("Unexpected error while parsing XML file. " + e.getMessage(), location);
}
}
@Override
public boolean isParseWithLimitsSupported() {
return true;
}
/**
* {@inheritDoc}
* @param maxBytes the maximum number of content bytes to process. Be careful with to small values :
* a Failure exception can eventually be thrown when maxBytes value is so small that the parser can even not fill its buffers on input stream and parse the document declaration.
*/
@Override
public Document[] parseWithLimits(DigestURL location, String mimeType, String charsetName, VocabularyScraper scraper,
int timezoneOffset, InputStream source, int maxLinks, long maxBytes)
throws Failure, InterruptedException, UnsupportedOperationException {
/* Limit the size of the in-memory buffer to at most 25% of the available memory :
* because some room is needed, and before being garbage collected the buffer will be converted to a String, then to a byte array.
* Eventual stricter limits should be handled by the caller (see for example crawler.[protocol].maxFileSize configuration setting). */
final long availableMemory = MemoryControl.available();
final long maxTextBytes = (long)(availableMemory * 0.25);
final int maxChars;
if((maxTextBytes / Character.BYTES) > Integer.MAX_VALUE) {
maxChars = Integer.MAX_VALUE;
} else {
maxChars = ((int)maxTextBytes) / Character.BYTES;
}
try (/* Automatically closed by this try-with-resources statement*/ CharBuffer writer = new CharBuffer(maxChars);){
final Set<AnchorURL> detectedURLs = new HashSet<>();
final GenericXMLContentHandler saxHandler = new GenericXMLContentHandler(writer, detectedURLs, maxLinks);
StrictLimitInputStream limitedSource = new StrictLimitInputStream(source, maxBytes);
/* Use commons-io XmlStreamReader advanced rules to help with charset detection when source contains no BOM or XML declaration
* (detection algorithm notably also include ContentType transmitted by HTTP headers, here eventually present as mimeType and charset parameters), */
final XmlStreamReader reader = new XmlStreamReader(limitedSource, mimeType, true, charsetName);
final InputSource saxSource = new InputSource(reader);
final String detectedCharset = reader.getEncoding();
final SAXParser saxParser = getParser();
boolean limitExceeded = false;
try {
saxParser.parse(saxSource, saxHandler);
} catch(SAXException e) {
if(!(e.getCause() instanceof SizeLimitExceededException)) {
/* Only transmit to upper layer exceptions that are not caused by the maxLinks limit being reached */
throw e;
}
limitExceeded = true;
} catch(StreamLimitException e) {
limitExceeded = true;
}
if (writer.isOverflow()) {
throw new Parser.Failure("Not enough Memory available for generic the XML parser : "
+ Formatter.bytesToString(availableMemory), location);
}
/* Create the parsed document with eventually only partial part of the text and links */
final byte[] contentBytes = UTF8.getBytes(writer.toString());
Document[] docs = new Document[] { new Document(location, mimeType, detectedCharset, this, null, null, null, null, "",
null, null, 0.0d, 0.0d, contentBytes, detectedURLs, null, null, false, new Date()) };
docs[0].setPartiallyParsed(limitExceeded);
return docs;
} catch (final Exception e) {
throw new Parser.Failure("Unexpected error while parsing XML file. " + e.getMessage(), location);
}
}
}
Mini Shell