Markov链:SQL数据库与Java表示
现在这个问题有点模糊。我有一个基于文本的马尔可夫链,它是通过解析用户类型的文本生成的。它用于生成几乎连贯的乱码字符串,并根据序列中的当前单词存储给定单词成为文本序列中下一个单词的概率。在javascript中,此对象的外观如下所示:Markov链:SQL数据库与Java表示,java,database,data-structures,text,markov-chains,Java,Database,Data Structures,Text,Markov Chains,现在这个问题有点模糊。我有一个基于文本的马尔可夫链,它是通过解析用户类型的文本生成的。它用于生成几乎连贯的乱码字符串,并根据序列中的当前单词存储给定单词成为文本序列中下一个单词的概率。在javascript中,此对象的外观如下所示: var text_markov_chain = { "apple" : { "cake" : 0.2, "sauce" : 0.8 }, "transformer" : { "movie" :
var text_markov_chain = {
"apple" : {
"cake" : 0.2,
"sauce" : 0.8
},
"transformer" : {
"movie" : 0.95,
"cat" : 0.025,
"dog" : 0.025
}
"cat" : {
"dog : 0.5,
"nap" : 0.5
}
// ...
}
例如,如果当前单词是transformer,那么我们生成的下一个单词将有95%的几率成为电影,2.5%的几率成为猫或狗
我的问题有两个:
- 用Java表示这个对象的最佳方式是什么?最好的情况是,我50%关心快速访问,50%关心内存使用
- 如何将此对象存储在单个数据库表中(例如MySQL)
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeMap;
public class MarkovChain {
HashMap<String, TreeMap<String, Float>> chain;
Set<String> known_words;
Random rand;
/**
* Generates a first order Markov Chain from the given text
* @param input_text The text to parse
*/
public MarkovChain(String input_text) {
init(input_text, 1);
}
/**
* Generates a nth order Markov Chain from the given text
* @param input_text The text to parse
* @param n The order of the Markov Chain
*/
public MarkovChain(String input_text, int n) {
init(input_text, n);
}
/**
* Reads a Markov Chain from the given input stream. The object is assumed
* to be binary and serialized
* @param in The input stream, eg from a network port or file
*/
public MarkovChain(InputStream in) {
try {
ObjectInputStream ob_in = new ObjectInputStream(in);
chain = (HashMap<String, TreeMap<String, Float>>)ob_in.readObject();
known_words = chain.keySet();
ob_in.close();
in.close();
} catch (IOException e) {
//e.printStackTrace();
chain = null;
known_words = null;
} catch (ClassNotFoundException e) {
//e.printStackTrace();
chain = null;
known_words = null;
}
}
/**
* Returns the next word, according to the Markov Chain probabilities
* @param current_word The current generated word
*/
public String nextWord(String current_word) {
if(current_word == null) return nextWord();
// Then head off down the yellow-markov-brick-road
TreeMap<String, Float> wordmap = chain.get(current_word);
if(wordmap == null) {
/* This *shouldn't* happen, but if we get a word that isn't in the
* Markov Chain, choose another random one
*/
return nextWord();
}
// Choose the next word based on an RV (Random Variable)
float rv = rand.nextFloat();
for(String word : wordmap.keySet()) {
float prob = wordmap.get(word);
rv -= prob;
if(rv <= 0) {
return word;
}
}
/* We should never get here - if we do, then the probabilities have
* been calculated incorrectly in the Markov Chain
*/
assert false : "Probabilities in Markov Chain must sum to one!";
return null;
}
/**
* Returns the next word when the current word is unknown, irrelevant or
* non existant (at the start of the sequence - randomly picks from known_words
*/
public String nextWord() {
return (String) known_words.toArray()[rand.nextInt(known_words.size())];
}
private void init(String input_text, int n) {
if(input_text.length() <= 0) return;
if(n <= 0) return;
chain = new HashMap<String, TreeMap<String, Float>>();
known_words = new HashSet<String>();
rand = new Random(new Date().getTime());
/** Generate the Markov Chain! **/
StringTokenizer st = new StringTokenizer(input_text);
while (st.hasMoreTokens()) {
String word = st.nextToken();
TreeMap<String, Float> wordmap = new TreeMap<String, Float>();
// First check if the current word has previously been parsed
if(known_words.contains(word)) continue;
known_words.add(word);
// Build the Markov probability table for this word
StringTokenizer st_this_word = new StringTokenizer(input_text);
String previous = "";
while (st_this_word.hasMoreTokens()) {
String next_word = st_this_word.nextToken();
if(previous.equals(word)) {
if(wordmap.containsKey(next_word)) {
// Increment the number of counts for this word by 1
float num = wordmap.get(next_word);
wordmap.put(next_word, num + 1);
} else {
wordmap.put(next_word, 1.0f);
}
}
previous = next_word;
} // End while (st_this_word.hasMoreTokens())
/* The wordmap now contains a map of words and the number of occurrences they have.
* We need to convert this to the probability of getting that word by dividing
* by the total number of words there were
*/
int total_number_of_words = wordmap.values().size();
for(String k : wordmap.keySet()) {
int num_occurances = wordmap.get(k).intValue();
wordmap.put(k, 1.0f*num_occurances/total_number_of_words);
}
// Finally, we are ready to add this word and wordmap to the Markov chain
chain.put(word, wordmap);
} // End while (st.hasMoreTokens())
// The (first order) Markov Chain has now been built!
}
}
import java.io.IOException;
导入java.io.InputStream;
导入java.io.ObjectInputStream;
导入java.util.Date;
导入java.util.HashMap;
导入java.util.HashSet;
导入java.util.Random;
导入java.util.Set;
导入java.util.StringTokenizer;
导入java.util.TreeMap;
公共类马尔可夫链{
HashMap链;
设置已知单词;
随机兰德;
/**
*从给定文本生成一阶马尔可夫链
*@param input_text要分析的文本
*/
公共MarkovChain(字符串输入\文本){
初始化(输入文本,1);
}
/**
*从给定文本生成N阶马尔可夫链
*@param input_text要分析的文本
*@param n马尔可夫链的顺序
*/
公共MarkovChain(字符串输入\文本,int n){
初始化(输入文本,n);
}
/**
*从给定的输入流中读取马尔可夫链。假定对象为
*要进行二进制和序列化
*输入流中的@param,如来自网络端口或文件
*/
公共MarkovChain(输入流输入){
试一试{
ObjectInputStream ob_in=新ObjectInputStream(in);
chain=(HashMap)ob_in.readObject();
已知单词=chain.keySet();
ob_in.close();
in.close();
}捕获(IOE异常){
//e、 printStackTrace();
链=空;
已知单词=空;
}catch(classnotfounde异常){
//e、 printStackTrace();
链=空;
已知单词=空;
}
}
/**
*根据马尔可夫链概率返回下一个单词
*@param current_word当前生成的字
*/
公共字符串nextWord(字符串当前单词){
if(current_word==null)返回nextWord();
//然后沿着黄砖路往前走
TreeMap wordmap=chain.get(当前单词);
if(wordmap==null){
/*这不应该发生,但如果我们得到一个不在
*马尔可夫链,选择另一个随机的
*/
返回nextWord();
}
//根据RV(随机变量)选择下一个单词
float rv=rand.nextFloat();
for(字符串字:wordmap.keySet()){
float prob=wordmap.get(word);
rv-=prob;
如果(rv将其存储在Java中,我猜您会考虑以一种易于生成序列的方式来存储它
首先你需要一个hashmap,单词是键。这个hashmap的值将是一个树映射,键是累积概率,值是下一个单词
因此,它将类似于:
HashMap<String, TreeMap<Double, String>> words = new HashMap<String, TreeMap<Double,String>>();
TreeMap<Double, String> appleMap = new TreeMap<Double, String>();
appleMap.put( 0.2d, "cake");
appleMap.put( 1.0d, "sauce");
words.put( "apple", appleMap );
TreeMap<Double, String> transformerMap = new TreeMap<Double, String>();
transformerMap.put( 0.95d, "movie");
transformerMap.put( 0.975d, "cat");
transformerMap.put( 1.0d, "dog");
words.put( "transformer", transformerMap );
HashMap words=newhashmap();
TreeMap appleMap=newtreemap();
应用映射放置(0.2d,“蛋糕”);
苹果酱(1.0d,“酱汁”);
单词.put(“苹果”,appleMap);
TreeMap transformerMap=newtreemap();
变形金刚地图(0.95d,“电影”);
变压器平均放电量(0.975d,“cat”);
变压器映射放置(1.0d,“狗”);
文字。put(“变压器”,变压器映射);
从这个结构很容易生成下一个单词
private String generateNextWord( HashMap<String, TreeMap<Double, String>> words, String currentWord ) {
TreeMap<Double, String> probMap = words.get( currentWord );
double d = Math.random();
return probMap.ceilingEntry( d ).getValue();
}
private String generateNextWord(HashMap字、String currentWord){
TreeMap probMap=words.get(currentWord);
double d=Math.random();
返回probMap.ceilingEntry(d.getValue();
}
在关系数据库中,您可以简单地拥有一个包含三列的表:当前字、下一个字和权重。因此,您基本上存储了马尔可夫链状态转移图的边
您还可以将其规范化为两个表:一个顶点表用于存储单词id对应的单词,另一个边缘表用于存储当前单词id、下一个单词id和权重,但除非您想用单词存储额外字段,否则我认为这是不必要的。我建议将其隐藏在sane类后面,以避免执行de操作每次客户端代码需要访问马尔可夫链时,都会使用映射。此外,d
前缀可以删除,因为Java中的浮点文本默认为double
@Sanjay T.Sharma好吧,通常你都会解析文件中的数据,我只是把init代码放进去传达这个想法。我个人更喜欢使用“d”或“f”后缀。我知道;我的评论是为了给你的文章添加额外的上下文。没有挑剔的意思。:-)谢谢你更新你的答案,包括关于databases@bizclop的内容!正是我想要的。