Current File : //home/waritko/yacy/source/net/yacy/document/parser/html/TransformerWriter.java
// htmlFilterOutputStream.java
// ---------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
/*
This class implements an output stream. Any data written to that output
is automatically parsed.
After finishing with writing, the htmlFilter can be read out.
*/
package net.yacy.document.parser.html;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.Enumeration;
import java.util.Locale;
import java.util.Properties;
import java.util.Stack;
import net.yacy.document.parser.html.ContentScraper.TagName;
import net.yacy.kelondro.io.CharBuffer;
public final class TransformerWriter extends Writer {
public static final char lb = '<';
public static final char rb = '>';
public static final char dash = '-';
public static final char excl = '!';
public static final char singlequote = '\'';
public static final char doublequote = '"';
private final OutputStream outStream;
private OutputStreamWriter out;
private CharBuffer buffer;
private Stack<ContentScraper.Tag> tagStack;
private final Scraper scraper;
private boolean inSingleQuote;
private boolean inDoubleQuote;
private boolean inComment;
private boolean binaryUnsuspect;
private final boolean passbyIfBinarySuspect;
public TransformerWriter(
final OutputStream outStream,
final Charset charSet,
final Scraper scraper,
final boolean passbyIfBinarySuspect
) {
this(outStream, charSet, scraper, passbyIfBinarySuspect, 64);
}
public TransformerWriter(
final OutputStream outStream,
final Charset charSet,
final Scraper scraper,
final boolean passbyIfBinarySuspect,
final int initialBufferSize
) {
this.outStream = outStream;
this.scraper = scraper;
this.buffer = new CharBuffer(ContentScraper.MAX_DOCSIZE, initialBufferSize);
this.tagStack = new Stack<ContentScraper.Tag>();
this.inSingleQuote = false;
this.inDoubleQuote = false;
this.inComment = false;
this.binaryUnsuspect = true;
this.passbyIfBinarySuspect = passbyIfBinarySuspect;
if (this.outStream != null) {
this.out = new OutputStreamWriter(this.outStream,(charSet == null)?Charset.defaultCharset():charSet);
}
}
public static char[] genTag0raw(final String tagname, final boolean opening, final char[] tagopts) {
final CharBuffer bb = new CharBuffer(ContentScraper.MAX_DOCSIZE, tagname.length() + tagopts.length + 3);
bb.append('<');
if (!opening) {
bb.append('/');
}
bb.append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
}
bb.append('>');
final char[] result = bb.getChars();
bb.close();
return result;
}
public static char[] genTag1raw(final String tagname, final char[] tagopts, final char[] text) {
final CharBuffer bb = new CharBuffer(ContentScraper.MAX_DOCSIZE, 2 * tagname.length() + tagopts.length + text.length + 5);
bb.append('<').append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
}
bb.append('>');
bb.append(text);
bb.append('<').append('/').append(tagname).append('>');
final char[] result = bb.getChars();
bb.close();
return result;
}
public static char[] genTag0(final String tagname, final Properties tagopts, final char quotechar) {
final char[] tagoptsx = (tagopts.isEmpty()) ? null : genOpts(tagopts, quotechar);
final CharBuffer bb = new CharBuffer(ContentScraper.MAX_DOCSIZE, tagname.length() + ((tagoptsx == null) ? 0 : (tagoptsx.length + 1)) + tagname.length() + 2);
bb.append('<').append(tagname);
if (tagoptsx != null) {
bb.appendSpace();
bb.append(tagoptsx);
}
bb.append('>');
final char[] result = bb.getChars();
bb.close();
return result;
}
public static char[] genTag1(final String tagname, final Properties tagopts, final char[] text, final char quotechar) {
final char[] gt0 = genTag0(tagname, tagopts, quotechar);
final CharBuffer cb = new CharBuffer(ContentScraper.MAX_DOCSIZE, gt0, gt0.length + text.length + tagname.length() + 3);
cb.append(text).append('<').append('/').append(tagname).append('>');
final char[] result = cb.getChars();
cb.close();
return result;
}
// a helper method for pretty-printing of properties for html tags
public static char[] genOpts(final Properties prop, final char quotechar) {
final Enumeration<?> e = prop.propertyNames();
final CharBuffer bb = new CharBuffer(ContentScraper.MAX_DOCSIZE, prop.size() * 40);
String key;
while (e.hasMoreElements()) {
key = (String) e.nextElement();
bb.appendSpace().append(key).append('=').append(quotechar);
bb.append(prop.getProperty(key));
bb.append(quotechar);
}
final char[] result;
if (bb.length() > 0)
result = bb.getChars(1);
else
result = bb.getChars();
bb.close();
return result;
}
/**
* the token processor distinguishes three different types of input: opening tag, closing tag, text content
* @param in - the token to be processed
* @param quotechar
* @return a processed version of the token
*/
private char[] tokenProcessor(final char[] in, final char quotechar) {
if (in.length == 0) return in;
// scan the string and parse structure
if (in.length <= 2 || in[0] != lb) return filterTag(in); // this is a text
// this is a tag
String tag;
int tagend;
if (in[1] == '/') {
// a closing tag
tagend = tagEnd(in, 2);
tag = new String(in, 2, tagend - 2).toLowerCase(Locale.ROOT);
final char[] text = new char[in.length - tagend - 1];
System.arraycopy(in, tagend, text, 0, in.length - tagend - 1);
return filterTag(text, quotechar, tag, false);
}
// don't add text from within <script> section, here e.g. a "if 1<a" expression could confuse tag detection
if (this.tagStack.size()>0 && this.tagStack.lastElement().name.equals(TagName.script.name())) {
return new char[0];
}
// an opening tag
tagend = tagEnd(in, 1);
tag = new String(in, 1, tagend - 1).toLowerCase(Locale.ROOT);
final char[] text = new char[in.length - tagend - 1];
System.arraycopy(in, tagend, text, 0, in.length - tagend - 1);
return filterTag(text, quotechar, tag, true);
}
// distinguish the following cases:
// - (1) not collecting data for a tag and getting no tag (not opener and not close)
// - (2) not collecting data for a tag and getting a tag opener
// - (3) not collecting data for a tag and getting a tag close
// - (4) collecting data for a tag and getting no tag (not opener and not close)
// - (5) collecting data for a tag and getting a new/different tag opener without closing the previous tag
// - (6) collecting data for a tag and getting a tag close for the wrong tag (a different than the opener)
// - (7) collecting data for a tag and getting the correct close tag for that collecting tag
/**
*
* @param content
* @return content or empty array
*/
private char[] filterTag(final char[] content) {
if (this.tagStack.size() == 0) {
// we are not collection tag text -> case (1) - (3)
// case (1): this is not a tag opener/closer
if (this.scraper != null && content.length > 0) {
this.scraper.scrapeText(content, null);
}
return content;
}
// we are collection tag text for the tag 'filterTag' -> case (4) - (7)
// case (4): getting no tag, go on collecting content
if (this.scraper != null) {
this.scraper.scrapeText(content, this.tagStack.lastElement());
}
this.tagStack.lastElement().content.append(content);
return new char[0];
}
private char[] filterTag(final char[] content, final char quotechar, final String tagname, final boolean opening) {
assert tagname != null;
if (this.tagStack.size() == 0) {
// we are not collection tag text -> case (1) - (3)
// we have a new tag
if (opening) {
// case (2):
return filterTagOpening(tagname, content);
}
// its a close tag where no should be
// case (3): we ignore that thing and return it again
return genTag0raw(tagname, false, content);
}
// we are collection tag text for the tag 'filterTag' -> case (4) - (7)
if (tagname.equals("!")) filterTag(content);
// it's a tag! which one?
if (opening) {
// case (5): the opening should not be here. But we keep the order anyway
this.tagStack.lastElement().content.append(filterTagOpening(tagname, content));
return new char[0];
}
if (!tagname.equalsIgnoreCase(this.tagStack.lastElement().name)) {
// case (6): its a closing tag, but the wrong one. just add it.
this.tagStack.lastElement().content.append(genTag0raw(tagname, opening, content));
return new char[0];
}
// it's our closing tag! return complete result.
return filterTagCloseing(quotechar);
}
private char[] filterTagOpening(final String tagname, final char[] content) {
final CharBuffer charBuffer = new CharBuffer(ContentScraper.MAX_DOCSIZE, content);
ContentScraper.Tag tag = new ContentScraper.Tag(tagname, charBuffer.propParser());
charBuffer.close();
final ContentScraper.Tag parentTag;
if(this.tagStack.size() > 0) {
parentTag = this.tagStack.lastElement();
} else {
parentTag = null;
}
/* Check scraper ignoring rules */
if (this.scraper != null && this.scraper.shouldIgnoreTag(tag, parentTag)) {
tag.setIgnore(true);
}
/* Apply processing relevant for any kind of tag opening */
if(this.scraper != null) {
this.scraper.scrapeAnyTagOpening(tag);
}
if (this.scraper != null && this.scraper.isTag0(tagname)) {
// this single tag is collected at once here
this.scraper.scrapeTag0(tag);
}
if (this.scraper != null && this.scraper.isTag1(tagname)) {
// ok, start collecting; we don't push this here to the scraper or transformer; we do that when the tag is closed.
this.tagStack.push(tag);
return new char[0];
}
// we ignore that thing and return it again
return genTag0raw(tagname, true, content);
}
private char[] filterTagCloseing(final char quotechar) {
char[] ret;
ContentScraper.Tag tag = this.tagStack.lastElement();
if (this.scraper != null) this.scraper.scrapeTag1(tag);
ret = genTag1(tag.name, tag.opts, tag.content.getChars(), quotechar);
if (this.scraper != null && this.scraper.isTag1(tag.name)) {
// remove the tag from the stack as soon as the tag is processed
this.tagStack.pop();
// at this point the characters from the recently processed tag must be attached to the previous tag
if (this.tagStack.size() > 0) this.tagStack.lastElement().content.append(ret);
}
return ret;
}
private char[] filterFinalize(final char quotechar) {
if (this.tagStack.size() == 0) {
return new char[0];
}
// it's our closing tag! return complete result.
char[] ret;
if (this.scraper != null) this.scraper.scrapeTag1(this.tagStack.lastElement());
ret = genTag1(this.tagStack.lastElement().name, this.tagStack.lastElement().opts, this.tagStack.lastElement().content.getChars(), quotechar);
this.tagStack.pop();
return ret;
}
private static int tagEnd(final char[] tag, final int start) {
char c;
for (int i = start; i < tag.length; i++) {
c = tag[i];
if (c != '!' && c != '-' &&
(c < '0' || c > '9') &&
(c < 'a' || c > 'z') &&
(c < 'A' || c > 'Z')
) return i;
}
return tag.length - 1;
}
/**
* this is the tokenizer of the parser: it splits the input into pieces which are
* - quoted text parts
* - commented text parts
* - tags (opening and closing)
* - text content between all these parts
* The tokens are then parsed with the filterSentence method
*/
@Override
public void write(final int c) throws IOException {
//System.out.println((char) c);
if ((this.binaryUnsuspect) && (binaryHint((char)c))) {
this.binaryUnsuspect = false;
if (this.passbyIfBinarySuspect) close();
}
if (this.binaryUnsuspect || !this.passbyIfBinarySuspect) {
char[] filtered;
if (this.inSingleQuote) {
this.buffer.append(c);
if (c == singlequote) this.inSingleQuote = false;
// check error cases
if ((c == rb) && (this.buffer.length() > 0 && this.buffer.charAt(0) == lb)) {
this.inSingleQuote = false;
// the tag ends here. after filtering: pass on
filtered = tokenProcessor(this.buffer.getChars(), singlequote);
if (this.out != null) { this.out.write(filtered); }
// this.buffer = new serverByteBuffer();
this.buffer.reset();
}
} else if (this.inDoubleQuote) {
this.buffer.append(c);
if (c == doublequote) this.inDoubleQuote = false;
// check error cases
if (c == rb && this.buffer.length() > 0 && this.buffer.charAt(0) == lb) {
this.inDoubleQuote = false;
// the tag ends here. after filtering: pass on
filtered = tokenProcessor(this.buffer.getChars(), doublequote);
if (this.out != null) this.out.write(filtered);
// this.buffer = new serverByteBuffer();
this.buffer.reset();
}
} else if (this.inComment) {
this.buffer.append(c);
if (c == rb &&
this.buffer.length() > 6 &&
this.buffer.charAt(this.buffer.length() - 3) == dash) {
// comment is at end
this.inComment = false;
final char[] comment = this.buffer.getChars();
if (this.scraper != null) this.scraper.scrapeComment(comment);
if (this.out != null) this.out.write(comment);
// this.buffer = new serverByteBuffer();
this.buffer.reset();
}
} else {
if (this.buffer.isEmpty()) {
if (c == rb) {
// very strange error case; we just let it pass
if (this.out != null) this.out.write(c);
} else {
this.buffer.append(c);
}
} else if (this.buffer.length() > 0 && this.buffer.charAt(0) == lb) {
if (c == singlequote) this.inSingleQuote = true;
if (c == doublequote) this.inDoubleQuote = true;
// fill in tag text
if ((this.buffer.length() >= 3) && (this.buffer.charAt(1) == excl) &&
(this.buffer.charAt(2) == dash) && (c == dash)) {
// this is the start of a comment
this.inComment = true;
this.buffer.append(c);
} else if (c == rb) {
this.buffer.append(c);
// the tag ends here. after filtering: pass on
filtered = tokenProcessor(this.buffer.getChars(), doublequote);
if (this.out != null) this.out.write(filtered);
// this.buffer = new serverByteBuffer();
this.buffer.reset();
} else if (c == lb) {
// this is an error case
// we consider that there is one rb missing
if (this.buffer.length() > 0) {
filtered = tokenProcessor(this.buffer.getChars(), doublequote);
if (this.out != null) this.out.write(filtered);
}
// this.buffer = new serverByteBuffer();
this.buffer.reset();
this.buffer.append(c);
} else {
this.buffer.append(c);
}
} else {
// fill in plain text
if (c == lb) {
// the text ends here
if (this.buffer.length() > 0) {
filtered = tokenProcessor(this.buffer.getChars(), doublequote);
if (this.out != null) this.out.write(filtered);
}
// this.buffer = new serverByteBuffer();
this.buffer.reset();
this.buffer.append(c);
} else {
// simply append
this.buffer.append(c);
}
}
}
} else {
this.out.write(c);
}
}
@Override
public void write(final char b[]) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(final char b[], final int off, final int len) throws IOException {
// System.out.println(UTF8.String(b, off, len));
if ((off | len | (b.length - (len + off)) | (off + len)) < 0) throw new IndexOutOfBoundsException();
for (int i = off ; i < (len - off) ; i++) this.write(b[i]);
}
@Override
public void flush() throws IOException {
// we cannot flush the current string this.buffer to prevent that
// the filter process is messed up
// instead, we simply flush the underlying output stream
if (this.out != null) this.out.flush();
if (this.scraper != null) this.scraper.finish();
// if you want to flush all, call close() at end of writing;
}
@Override
public void close() throws IOException {
flush();
final char quotechar = (this.inSingleQuote) ? singlequote : doublequote;
if (this.buffer != null) {
if (this.buffer.length() > 0) {
final char[] filtered = tokenProcessor(this.buffer.getChars(), quotechar);
if (this.out != null) this.out.write(filtered);
}
this.buffer.close();
this.buffer = null;
}
final char[] finalized = filterFinalize(quotechar);
if (this.out != null) {
if (finalized != null) this.out.write(finalized);
this.out.flush();
this.out.close();
}
this.tagStack.clear();
this.tagStack = null;
if (this.scraper != null) this.scraper.finish();
}
private static boolean binaryHint(final char c) {
// space, punctiation and symbols, letters and digits (ASCII/latin)
//if (c >= 31 && c < 128) return false;
if(c > 31) return false;
// 8 = backspace
// 9 = horizontal tab
// 10 = new line (line feed)
// 11 = vertical tab
// 12 = new page (form feed)
// 13 = carriage return
if (c > 7 && c <= 13) return false;
//if (Character.isLetterOrDigit(c)) return false;
// return false;
// System.err.println("BINARY HINT: " + (int) c);
return true;
}
public boolean binarySuspect() {
return !this.binaryUnsuspect;
}
}
Mini Shell