Current File : //home/waritko/yacy/source/net/yacy/cora/document/analysis/Classification.java
/**
* Classification.java
* Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 09.07.2009 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.document.analysis;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.util.CommonPattern;
public class Classification {
private static final Set<String> textExtSet = new HashSet<String>();
private static final Set<String> mediaExtSet = new HashSet<String>();
private static final Set<String> imageExtSet = new HashSet<String>();
private static final Set<String> audioExtSet = new HashSet<String>();
private static final Set<String> videoExtSet = new HashSet<String>();
private static final Set<String> appsExtSet = new HashSet<String>();
private static final Set<String> ctrlExtSet = new HashSet<String>();
public enum ContentDomain {
ALL(-1),
TEXT(0),
IMAGE(1),
AUDIO(2),
VIDEO(3),
APP(4),
CTRL(5);
private final int code;
ContentDomain(int code) {
this.code = code;
}
public int getCode() {
return this.code;
}
public static ContentDomain contentdomParser(final String dom) {
if ("all".equals(dom)) return ALL;
else if ("text".equals(dom)) return TEXT;
else if ("image".equals(dom)) return IMAGE;
else if ("audio".equals(dom)) return AUDIO;
else if ("video".equals(dom)) return VIDEO;
else if ("app".equals(dom)) return APP;
else if ("ctrl".equals(dom)) return CTRL;
return TEXT;
}
@Override
public String toString() {
if (this == ALL) return "all";
else if (this == TEXT) return "text";
else if (this == IMAGE) return "image";
else if (this == AUDIO) return "audio";
else if (this == VIDEO) return "video";
else if (this == APP) return "app";
else if (this == CTRL) return "ctrl";
return "text";
}
}
static {
final String text = "htm,html,phtml,shtml,xhtml,php,php3,php4,php5,cfm,asp,aspx,tex,txt,jsp,mf,asp,aspx,csv,gpx,vcf,xsl,xml,pdf,doc,docx,xls,xlsx,ppt,pptx";
final String apps = "7z,ace,arc,arj,apk,asf,asx,bat,bin,bkf,bz2,cab,com,css,dcm,deb,dll,dmg,exe,java,gho,ghs,gz,hqx,img,iso,jar,lha,rar,sh,sit,sitx,tar,tbz,tgz,tib,torrent,vbs,war,zip";
final String audio = "aac,aif,aiff,aifc,flac,m4a,m4b,m4p,mid,mp2,mp3,oga,ogg,ra,ram,sid,wav,wma";
final String video = "3g2,3gp,3gp2,3gpp,3gpp2,3ivx,asf,asx,avi,div,divx,dv,dvx,env,f4v,flv,hdmov,m1v,m4v,m-jpeg,mkv,moov,mov,movie,mp2v,mp4,mpe,mpeg,mpg,mpg4,mv4,ogm,ogv,qt,rm,rv,vid,swf,webm,wmv";
final String image = "ai,bmp,cdr,cmx,emf,eps,gif,img,jpeg,jpg,mng,pct,pdd,pdn,pict,png,psb,psd,psp,svg,tif,tiff,wmf";
final String ctrl = "sha1,md5,crc32,sfv";
addSet(textExtSet, text); // text formats
addSet(imageExtSet, image); // image formats
addSet(audioExtSet, audio); // audio formats
addSet(videoExtSet, video); // video formats
addSet(appsExtSet, apps); // application formats
addSet(ctrlExtSet, ctrl); // control formats
addSet(mediaExtSet, apps + "," + audio + "," + video + "," + image); // all media formats
}
private static void addSet(Set<String> set, final String extString) {
if ((extString == null) || (extString.isEmpty())) return;
for (String s: CommonPattern.COMMA.split(extString, 0)) set.add(s.toLowerCase(Locale.ROOT).trim());
}
public static boolean isTextExtension(String textExt) {
if (textExt == null) return false;
return textExtSet.contains(textExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isMediaExtension(String mediaExt) {
if (mediaExt == null) return false;
return mediaExtSet.contains(mediaExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isImageExtension(final String imageExt) {
if (imageExt == null) return false;
return imageExtSet.contains(imageExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isAudioExtension(final String audioExt) {
if (audioExt == null) return false;
return audioExtSet.contains(audioExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isVideoExtension(final String videoExt) {
if (videoExt == null) return false;
return videoExtSet.contains(videoExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isApplicationExtension(final String appsExt) {
if (appsExt == null) return false;
return appsExtSet.contains(appsExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isControlExtension(final String ctrlExt) {
if (ctrlExt == null) return false;
return ctrlExtSet.contains(ctrlExt.trim().toLowerCase(Locale.ROOT));
}
public static boolean isAnyKnownExtension(String ext) {
if (ext == null) return false;
ext = ext.trim().toLowerCase(Locale.ROOT);
return textExtSet.contains(ext) || mediaExtSet.contains(ext) || ctrlExtSet.contains(ext);
}
/**
* Get the content domain of a document according to the file extension.
* This can produce wrong results because the extension is a weak hint for the content domain.
* If possible, use the mime type, call Classification.getContentDomainFromMime()
* @return the content domain which classifies the content type
*/
public static ContentDomain getContentDomainFromExt(final String ext) {
if (isTextExtension(ext)) return ContentDomain.TEXT;
if (isImageExtension(ext)) return ContentDomain.IMAGE;
if (isAudioExtension(ext)) return ContentDomain.AUDIO;
if (isVideoExtension(ext)) return ContentDomain.VIDEO;
if (isApplicationExtension(ext)) return ContentDomain.APP;
if (isControlExtension(ext)) return ContentDomain.CTRL;
return ContentDomain.ALL;
}
/**
* Get the content domain of a document according to the mime type.
* @return the content domain which classifies the content type
*/
public static ContentDomain getContentDomainFromMime(final String mime) {
if (mime != null) {
if (mime.startsWith("text/")) return ContentDomain.TEXT;
if (mime.startsWith("image/")) return ContentDomain.IMAGE;
if (mime.startsWith("audio/")) return ContentDomain.AUDIO;
if (mime.startsWith("video/")) return ContentDomain.VIDEO;
if (mime.startsWith("application/")) return ContentDomain.APP;
}
return ContentDomain.ALL;
}
public static boolean isPictureMime(final String mimeType) {
if (mimeType == null) return false;
return mimeType.toUpperCase(Locale.ROOT).startsWith("IMAGE");
}
private static final Properties mimeTable = new Properties();
public static void init(final File mimeFile) {
if (mimeTable.isEmpty()) {
// load the mime table
BufferedInputStream mimeTableInputStream = null;
try {
mimeTableInputStream = new BufferedInputStream(new FileInputStream(mimeFile));
mimeTable.load(mimeTableInputStream);
} catch (final Exception e) {
} finally {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (final Exception e1) {}
}
}
for (Entry<Object, Object> entry: mimeTable.entrySet()) {
String ext = (String) entry.getKey();
String mime = (String) entry.getValue();
if (mime.startsWith("text/")) textExtSet.add(ext.toLowerCase(Locale.ROOT));
if (mime.startsWith("audio/")) audioExtSet.add(ext.toLowerCase(Locale.ROOT));
if (mime.startsWith("video/")) videoExtSet.add(ext.toLowerCase(Locale.ROOT));
if (mime.startsWith("application/")) appsExtSet.add(ext.toLowerCase(Locale.ROOT));
}
}
public static int countMimes() {
return mimeTable.size();
}
public static String ext2mime(final String ext) {
return ext == null ? "application/octet-stream" : mimeTable.getProperty(ext.toLowerCase(Locale.ROOT), "application/" + (ext == null || ext.length() == 0 ? "octet-stream" : ext));
}
public static String ext2mime(final String ext, final String dfltMime) {
return ext == null ? dfltMime : mimeTable.getProperty(ext.toLowerCase(Locale.ROOT), dfltMime);
}
public static String url2mime(final MultiProtocolURL url, final String dfltMime) {
return url == null ? dfltMime : ext2mime(MultiProtocolURL.getFileExtension(url.getFileName()), dfltMime);
}
public static String url2mime(final MultiProtocolURL url) {
return url == null ? "application/octet-stream" : ext2mime(MultiProtocolURL.getFileExtension(url.getFileName()));
}
}
Mini Shell