利用文本挖掘技术来找出网络中的“小鲜词”
发布时间:2021-01-20 19:27:41 所属栏目:大数据 来源:网络整理
导读:开始之前,先看一下从人人网中发现的90后用户爱用的词 是不是很好玩,哈哈。写这篇文章就是让你简单的自动的从文本中找出新的词,这样就知道现在的年轻人喜欢什么了(对于博主这种上了年纪的人来说,真的是很有用,呜呜) 项目结构 当然,text.dat和common.d
这几个类用于给词创建索引,方便从词典中找出 CnPreviewTextIndexer.javapackage grid.text.index; import grid.common.TextUtils; import java.util.HashMap; import java.util.Map; import java.util.Vector; public class CnPreviewTextIndexer implements TextIndexer { private final static int CN_LETTER_COUNT = 5021; private String document; private Map<Character,Vector<Integer>> posMap; public CnPreviewTextIndexer(String document) { this.document = document; init(); } private void init() { final int len = document.length(); final int supposedMinCount = 1 + (int) Math.log(len / CN_LETTER_COUNT + 1); char c; Vector<Integer> posVector; posMap = new HashMap<Character,Vector<Integer>>(CN_LETTER_COUNT); for (int i = 0; i < len; i++) { c = document.charAt(i); if (!TextUtils.isCnLetter(c)) { continue; } posVector = posMap.get(c); if (null == posVector) { posVector = new Vector<Integer>(supposedMinCount); posMap.put(c,posVector); } posVector.add(i); } } @Override public int count(String text) { if (TextUtils.isBlank(text)) { return 0; } Vector<Integer> vector = posMap.get(text.charAt(0)); if (null == vector) { return 0; } if (1 == text.length()) { return vector.size(); } final int size = vector.size(); int count = 0; for (int i = 0; i < size; i++) { if (TextUtils.match(document,vector.get(i),text)) { count++; } } return count; } @Override public Pos find(Pos pos) { String text = pos.getTarget(); pos.setFound(false); if (TextUtils.isBlank(text)) { return pos; } Vector<Integer> vector = posMap.get(text.charAt(0)); if (null == vector) { return pos; } final int arraySize = vector.size(); final int arrayIndex = pos.arrayIndex + 1; for (int i = arrayIndex; i < arraySize; i++) { if (TextUtils.match(document,text)) { pos.setFound(true); pos.setPos(vector.get(i)); pos.arrayIndex = i; break; } } return pos; } @Override public int len() { return document.length(); } @Override public String sub(int off,int len) { if (off < 0 || off + len >= document.length()) { return ""; } return document.substring(off,off + len); } @Override public char charAt(int index) { if (index < 0 || index >= document.length()) { return 0; } return document.charAt(index); } } Pos.javapackage grid.text.index; public class Pos { private String target; /** * Pos for current matched full target text */ private int pos = -1; /** * Index in position array for current matched full target text */ int arrayIndex = -1; private boolean found = false; public Pos(String target) { this.target = target; } public String getTarget() { return target; } public int getPos() { return pos; } public boolean isFound() { return found; } void setPos(int pos) { this.pos = pos; } void setFound(boolean found) { this.found = found; } } SimpleTextIndexer.javapackage grid.text.index; public class SimpleTextIndexer implements TextIndexer { private String document; public SimpleTextIndexer(String document) { this.document = document; } @Override public int count(String text) { int off = 0; int count = 0; final int len = text.length(); while ((off = document.indexOf(text,off)) > -1) { count++; off += len; } return count; } @Override public Pos find(Pos pos) { final String text = pos.getTarget(); final int len = text.length(); int off = pos.getPos() + len; if (pos.getPos() < 0) off = 0; pos.setFound(false); if ((off = document.indexOf(text,off)) > -1) { pos.setFound(true); pos.setPos(off); } return pos; } @Override public int len() { return document.length(); } @Override public String sub(int off,int len) { return document.substring(off,off + len); } @Override public char charAt(int index) { if (index < 0 || index >= document.length()) { return 0; } return document.charAt(index); } } TextIndexer.javapackage grid.text.index; public interface TextIndexer { /** * @param text * @return count for specific text */ public int count(String text); /** * @param pos * @return next position for current pos */ public Pos find(Pos pos); /** * @return original document length */ public int len(); /** * @param off * @param len * @return the sub string start from <b>off</b> and with a length with * <b>len</b> */ public String sub(int off,int len); /** * @param index * @return return the character in the specified index */ public char charAt(int index); } participle(编辑:晋中站长网) 【声明】本站内容均来自网络,其相关言论仅代表作者个人观点,不代表本站立场。若无意侵犯到您的权利,请及时与联系站长删除相关内容! |