- 浏览: 31925 次
最新评论
TF-IDF(转)输出到文本
- 博客分类:
- 特征提取
import java.io.*;
import java.util.*;
import org.wltea.analyzer.lucene.IKAnalyzer;
public class ReadFiles {
/**
* @param args
*/
private static ArrayList<String> FileList = new ArrayList<String>(); // the list of file
//get list of file for the directory, including sub-directory of it
public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException
{
try
{
File file = new File(filepath);
if(!file.isDirectory())
{
System.out.println("输入的[]");
System.out.println("filepath:" + file.getAbsolutePath());
}
else
{
String[] flist = file.list();
for(int i = 0; i < flist.length; i++)
{
File newfile = new File(filepath + "\\" + flist[i]);
if(!newfile.isDirectory())
{
FileList.add(newfile.getAbsolutePath());
}
else if(newfile.isDirectory()) //if file is a directory, call ReadDirs
{
readDirs(filepath + "\\" + flist[i]);
}
}
}
}catch(FileNotFoundException e)
{
System.out.println(e.getMessage());
}
return FileList;
}
//read file
public static String readFile(String file) throws FileNotFoundException, IOException
{
StringBuffer strSb = new StringBuffer(); //String is constant, StringBuffer can be changed.
InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk"); //byte streams to character streams
BufferedReader br = new BufferedReader(inStrR);
String line = br.readLine();
while(line != null){
strSb.append(line).append("\r\n");
line = br.readLine();
}
return strSb.toString();
}
//word segmentation
public static ArrayList<String> cutWords(String file) throws IOException{
ArrayList<String> words = new ArrayList<String>();
String text = ReadFiles.readFile(file);
IKAnalyzer analyzer = new IKAnalyzer();
words = analyzer.split(text);
return words;
}
//term frequency in a file, times for each word
public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords){
HashMap<String, Integer> resTF = new HashMap<String, Integer>();
for(String word : cutwords){
if(resTF.get(word) == null){
resTF.put(word, 1);
System.out.println(word);
}
else{
resTF.put(word, resTF.get(word) + 1);
System.out.println(word.toString());
}
}
return resTF;
}
//term frequency in a file, frequency of each word
public static HashMap<String, Float> tf(ArrayList<String> cutwords){
HashMap<String, Float> resTF = new HashMap<String, Float>();
int wordLen = cutwords.size();
HashMap<String, Integer> intTF = ReadFiles.normalTF(cutwords);
Iterator iter = intTF.entrySet().iterator(); //iterator for that get from TF
try
{
FileWriter writer = new FileWriter("d:\\DF.txt", true);
while(iter.hasNext()){
Map.Entry entry = (Map.Entry)iter.next();
resTF.put(entry.getKey().toString(), Float.parseFloat(entry.getValue().toString()) / wordLen);
System.out.println(entry.getKey().toString() + " = "+ Float.parseFloat(entry.getValue().toString()) / wordLen);
// 输出到文件
writer.write(entry.getKey().toString() + " = "+ Float.parseFloat(entry.getValue().toString()) / wordLen+"\r\n");
}//end with while
writer.close();
}
catch(Exception ex)
{
}
return resTF;
}
//tf times for file
public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException{
HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String,Integer>>();
List<String> filelist = ReadFiles.readDirs(dirc);
for(String file : filelist){
HashMap<String, Integer> dict = new HashMap<String, Integer>();
ArrayList<String> cutwords = ReadFiles.cutWords(file); //get cut word for one file
dict = ReadFiles.normalTF(cutwords);
allNormalTF.put(file, dict);
}
return allNormalTF;
}
//tf for all file
public static HashMap<String,HashMap<String, Float>> tfAllFiles(String dirc) throws IOException{
HashMap<String, HashMap<String, Float>> allTF = new HashMap<String, HashMap<String, Float>>();
List<String> filelist = ReadFiles.readDirs(dirc);
for(String file : filelist){
HashMap<String, Float> dict = new HashMap<String, Float>();
ArrayList<String> cutwords = ReadFiles.cutWords(file); //get cut words for one file
dict = ReadFiles.tf(cutwords);
allTF.put(file, dict);
}
return allTF;
}
public static HashMap<String, Float> idf(HashMap<String,HashMap<String, Float>> all_tf){
HashMap<String, Float> resIdf = new HashMap<String, Float>();
HashMap<String, Integer> dict = new HashMap<String, Integer>();
int docNum = FileList.size();
for(int i = 0; i < docNum; i++){
HashMap<String, Float> temp = all_tf.get(FileList.get(i));
Iterator iter = temp.entrySet().iterator();
while(iter.hasNext()){
Map.Entry entry = (Map.Entry)iter.next();
String word = entry.getKey().toString();
if(dict.get(word) == null){
dict.put(word, 1);
}else {
dict.put(word, dict.get(word) + 1);
}
}
}
System.out.println("IDF for every word is:");
try
{
FileWriter writer = new FileWriter("d:\\IDF.txt", true);
Iterator iter_dict = dict.entrySet().iterator();
while(iter_dict.hasNext()){
Map.Entry entry = (Map.Entry)iter_dict.next();
float value = (float)Math.log(docNum / Float.parseFloat(entry.getValue().toString()));
resIdf.put(entry.getKey().toString(), value);
System.out.println(entry.getKey().toString() + " = " + value);
writer.write(entry.getKey().toString() + " = " + value+"\r\n");
}
writer.close();
}
catch(Exception ex)
{
System.out.println("Error");
return null;
}
return resIdf;
}
public static void tf_idf(HashMap<String,HashMap<String, Float>> all_tf,HashMap<String, Float> idfs){
HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>();
int docNum = FileList.size();
for(int i = 0; i < docNum; i++){
String filepath = FileList.get(i);
HashMap<String, Float> tfidf = new HashMap<String, Float>();
HashMap<String, Float> temp = all_tf.get(filepath);
Iterator iter = temp.entrySet().iterator();
while(iter.hasNext()){
Map.Entry entry = (Map.Entry)iter.next();
String word = entry.getKey().toString();
Float value = (float)Float.parseFloat(entry.getValue().toString()) * idfs.get(word);
tfidf.put(word, value);
}
resTfIdf.put(filepath, tfidf);
}
System.out.println("TF-IDF for Every file is :");
DisTfIdf(resTfIdf);
}
public static void DisTfIdf(HashMap<String, HashMap<String, Float>> tfidf){
Iterator iter1 = tfidf.entrySet().iterator();
try
{
FileWriter writer = new FileWriter("d:\\TF-IDF.txt", true);
String strtemp="";
while(iter1.hasNext()){
Map.Entry entrys = (Map.Entry)iter1.next();
System.out.println("FileName: " + entrys.getKey().toString());
//writer.write("FileName: " + entrys.getKey().toString());
System.out.print("{");
writer.write("{");
HashMap<String, Float> temp = (HashMap<String, Float>) entrys.getValue();
Iterator iter2 = temp.entrySet().iterator();
while(iter2.hasNext()){
Map.Entry entry = (Map.Entry)iter2.next();
System.out.print(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
// 输出到文件
strtemp+=entry.getKey().toString() + " = " + entry.getValue().toString() + ", ";
//writer.write(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
}
strtemp=strtemp.substring(0, strtemp.length()-2);
writer.write(strtemp);
System.out.println("}");
writer.write("}"+"\r\n");
}
writer.close();
}
catch(Exception ex)
{
System.out.println("error!");
return;
}
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String file = "D:/testfiles";
HashMap<String,HashMap<String, Float>> all_tf = tfAllFiles(file);
System.out.println();
HashMap<String, Float> idfs = idf(all_tf);
System.out.println();
tf_idf(all_tf, idfs);
}
}
import java.util.*;
import org.wltea.analyzer.lucene.IKAnalyzer;
public class ReadFiles {
/**
* @param args
*/
private static ArrayList<String> FileList = new ArrayList<String>(); // the list of file
//get list of file for the directory, including sub-directory of it
public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException
{
try
{
File file = new File(filepath);
if(!file.isDirectory())
{
System.out.println("输入的[]");
System.out.println("filepath:" + file.getAbsolutePath());
}
else
{
String[] flist = file.list();
for(int i = 0; i < flist.length; i++)
{
File newfile = new File(filepath + "\\" + flist[i]);
if(!newfile.isDirectory())
{
FileList.add(newfile.getAbsolutePath());
}
else if(newfile.isDirectory()) //if file is a directory, call ReadDirs
{
readDirs(filepath + "\\" + flist[i]);
}
}
}
}catch(FileNotFoundException e)
{
System.out.println(e.getMessage());
}
return FileList;
}
//read file
public static String readFile(String file) throws FileNotFoundException, IOException
{
StringBuffer strSb = new StringBuffer(); //String is constant, StringBuffer can be changed.
InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk"); //byte streams to character streams
BufferedReader br = new BufferedReader(inStrR);
String line = br.readLine();
while(line != null){
strSb.append(line).append("\r\n");
line = br.readLine();
}
return strSb.toString();
}
//word segmentation
public static ArrayList<String> cutWords(String file) throws IOException{
ArrayList<String> words = new ArrayList<String>();
String text = ReadFiles.readFile(file);
IKAnalyzer analyzer = new IKAnalyzer();
words = analyzer.split(text);
return words;
}
//term frequency in a file, times for each word
public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords){
HashMap<String, Integer> resTF = new HashMap<String, Integer>();
for(String word : cutwords){
if(resTF.get(word) == null){
resTF.put(word, 1);
System.out.println(word);
}
else{
resTF.put(word, resTF.get(word) + 1);
System.out.println(word.toString());
}
}
return resTF;
}
//term frequency in a file, frequency of each word
public static HashMap<String, Float> tf(ArrayList<String> cutwords){
HashMap<String, Float> resTF = new HashMap<String, Float>();
int wordLen = cutwords.size();
HashMap<String, Integer> intTF = ReadFiles.normalTF(cutwords);
Iterator iter = intTF.entrySet().iterator(); //iterator for that get from TF
try
{
FileWriter writer = new FileWriter("d:\\DF.txt", true);
while(iter.hasNext()){
Map.Entry entry = (Map.Entry)iter.next();
resTF.put(entry.getKey().toString(), Float.parseFloat(entry.getValue().toString()) / wordLen);
System.out.println(entry.getKey().toString() + " = "+ Float.parseFloat(entry.getValue().toString()) / wordLen);
// 输出到文件
writer.write(entry.getKey().toString() + " = "+ Float.parseFloat(entry.getValue().toString()) / wordLen+"\r\n");
}//end with while
writer.close();
}
catch(Exception ex)
{
}
return resTF;
}
//tf times for file
public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException{
HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String,Integer>>();
List<String> filelist = ReadFiles.readDirs(dirc);
for(String file : filelist){
HashMap<String, Integer> dict = new HashMap<String, Integer>();
ArrayList<String> cutwords = ReadFiles.cutWords(file); //get cut word for one file
dict = ReadFiles.normalTF(cutwords);
allNormalTF.put(file, dict);
}
return allNormalTF;
}
//tf for all file
public static HashMap<String,HashMap<String, Float>> tfAllFiles(String dirc) throws IOException{
HashMap<String, HashMap<String, Float>> allTF = new HashMap<String, HashMap<String, Float>>();
List<String> filelist = ReadFiles.readDirs(dirc);
for(String file : filelist){
HashMap<String, Float> dict = new HashMap<String, Float>();
ArrayList<String> cutwords = ReadFiles.cutWords(file); //get cut words for one file
dict = ReadFiles.tf(cutwords);
allTF.put(file, dict);
}
return allTF;
}
public static HashMap<String, Float> idf(HashMap<String,HashMap<String, Float>> all_tf){
HashMap<String, Float> resIdf = new HashMap<String, Float>();
HashMap<String, Integer> dict = new HashMap<String, Integer>();
int docNum = FileList.size();
for(int i = 0; i < docNum; i++){
HashMap<String, Float> temp = all_tf.get(FileList.get(i));
Iterator iter = temp.entrySet().iterator();
while(iter.hasNext()){
Map.Entry entry = (Map.Entry)iter.next();
String word = entry.getKey().toString();
if(dict.get(word) == null){
dict.put(word, 1);
}else {
dict.put(word, dict.get(word) + 1);
}
}
}
System.out.println("IDF for every word is:");
try
{
FileWriter writer = new FileWriter("d:\\IDF.txt", true);
Iterator iter_dict = dict.entrySet().iterator();
while(iter_dict.hasNext()){
Map.Entry entry = (Map.Entry)iter_dict.next();
float value = (float)Math.log(docNum / Float.parseFloat(entry.getValue().toString()));
resIdf.put(entry.getKey().toString(), value);
System.out.println(entry.getKey().toString() + " = " + value);
writer.write(entry.getKey().toString() + " = " + value+"\r\n");
}
writer.close();
}
catch(Exception ex)
{
System.out.println("Error");
return null;
}
return resIdf;
}
public static void tf_idf(HashMap<String,HashMap<String, Float>> all_tf,HashMap<String, Float> idfs){
HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>();
int docNum = FileList.size();
for(int i = 0; i < docNum; i++){
String filepath = FileList.get(i);
HashMap<String, Float> tfidf = new HashMap<String, Float>();
HashMap<String, Float> temp = all_tf.get(filepath);
Iterator iter = temp.entrySet().iterator();
while(iter.hasNext()){
Map.Entry entry = (Map.Entry)iter.next();
String word = entry.getKey().toString();
Float value = (float)Float.parseFloat(entry.getValue().toString()) * idfs.get(word);
tfidf.put(word, value);
}
resTfIdf.put(filepath, tfidf);
}
System.out.println("TF-IDF for Every file is :");
DisTfIdf(resTfIdf);
}
public static void DisTfIdf(HashMap<String, HashMap<String, Float>> tfidf){
Iterator iter1 = tfidf.entrySet().iterator();
try
{
FileWriter writer = new FileWriter("d:\\TF-IDF.txt", true);
String strtemp="";
while(iter1.hasNext()){
Map.Entry entrys = (Map.Entry)iter1.next();
System.out.println("FileName: " + entrys.getKey().toString());
//writer.write("FileName: " + entrys.getKey().toString());
System.out.print("{");
writer.write("{");
HashMap<String, Float> temp = (HashMap<String, Float>) entrys.getValue();
Iterator iter2 = temp.entrySet().iterator();
while(iter2.hasNext()){
Map.Entry entry = (Map.Entry)iter2.next();
System.out.print(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
// 输出到文件
strtemp+=entry.getKey().toString() + " = " + entry.getValue().toString() + ", ";
//writer.write(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
}
strtemp=strtemp.substring(0, strtemp.length()-2);
writer.write(strtemp);
System.out.println("}");
writer.write("}"+"\r\n");
}
writer.close();
}
catch(Exception ex)
{
System.out.println("error!");
return;
}
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String file = "D:/testfiles";
HashMap<String,HashMap<String, Float>> all_tf = tfAllFiles(file);
System.out.println();
HashMap<String, Float> idfs = idf(all_tf);
System.out.println();
tf_idf(all_tf, idfs);
}
}
相关推荐
tfidf_train()函数用于训练TF-IDF特征提取器,并将特征提取器保存到磁盘。 tfidf_test()函数用于加载保存在磁盘上的TF-IDF特征提取器,并使用它来处理测试数据。 svm_grid()函数用于使用网格搜索法寻找最佳的支持...
用java编写的tf*idf 结果输出txt文本,方便作后来的聚类矩阵
在我们得到词频(TF)和逆文档频率(IDF)以后,将两个值相乘,即可得到一个词的TF-IDF值,某个词对文章的重要性越高,其TF-IDF值就越大,所以排在最前面的几个词就是文章的关键词。 TF-IDF算法的优点是简单快速,...
输入那些,您将获得关联规则作为输出。 就是这样。 做得好! 先决条件 需要在计算机上安装python 3.6。 运行测试 编写代码的方式是,当您执行TextMining.py时,它将检查名为documentDatabase的文件夹并读取其中的...
TF-IDF TF-IDF(Term Frequencey-Inverse Document Frequency)指词频-逆文档频率,它属于数值统计的范畴。使用TF-IDF,我们能够学习一个词对于数据集中的一个文档的重要性。 TF-IDF的概念 TF-IDF有两部分,词频和逆...
1、将文件夹中所有的txt文件中的内容,按行读取,每一行作为一个post,对每一个post以所有txt中的内容作为全体计算tf-idf,输出为与原始txt及每一行对应的tf-idf。由于结果包含大量的0,所以采取了稀疏矩阵的存储...
开发一款针对英文文本的信息检索系统,可以实现建立索引表、布尔...(3)计算指定词的 TF-IDF 值; (4)进行布尔查询; (5)进行通配符查询; (6)进行短语查询。 所有功能都可以通过—hit 参数限制输出的结果数量。
【实验目的】: 开发一款针对英文文本的信息检索系统,可以实现建立索引表、布尔查询、...计算指定词的 TF-IDF值; 进行布尔查询; 进行通配符查询; 进行短语查询。 所有功能都可以通过—hit参数限制输出的结果数量。
数模转换 ...常见的数模转换包括词袋模型、TF-IDF向量化、词嵌入(Word Embedding)等。 4. **传感器数据处理**:传感器通常输出模拟信号,需要将其转换为数字信号并进行特征提取,以便输入到机器学习
将输入文本中的词通过编码映射到词嵌入矩阵中, 词向量特征经嵌入和平均叠加后, 和基于TF-IDF的文本向量特征进行拼接, 传入到输出层后计算属于每个分类的概率. 该模型在低维词向量的基础上结合了文本向量特征的表达...
* SVM:支持向量机模型,基于TF-IDF或wordvec * LR:逻辑回归模型,基于TF-IDF或wordvec * Stack-Propagation:https://aclanthology.org/D19-1214.pdf * Bi-model with decoder:...
TF-IDF中TF的定义如下: ![](media/8659ed935357d383513379963bac3424.png) IDF的定义如下: ![](media/4bef51dfd171cb29ab3f98dfdd9af41d.png) 我们对数据进行了一些预处理,主要包括:1)删除轮次低于3的会话...
(3) 对于分词后的数据信息进行词频统计,采用TF-IDF词频统计方法,将结果保存到本地。同时输出纯词频个数统计并保存。 (4) 利用词频数据对文本信息进行向量化,建立数据矩阵,并保存。 (5) 采用k-means聚类...
使用tfidf文件夹中的语料库计算一个bigram列表,并将此列表用作术语列表以计算tf-idf值并将结果输出到excel文件tfidf_result 识别具有相似性的相似文件 使用tfidf文件夹中的语料库来识别与doc_0.txt最相似的5个文档...
我们提供了几种数据预处理方法:BoW(单词袋),TF-IDF,word2vec,doc2vec。 每个py文件都会生成x_1(文档表示形式)x_2(标题表示形式)和y(标签)。 这些数据可以作为间谍数据输出,可以在模型中使用。 2.常规...
令牌被馈送到TF-IDF转换器,然后馈入机器学习分类器。 初始分类器模型(请参见下面的讨论)采用了非常简单的Complement Naive Bayees分类器方法,该方法因其在不平衡类中的首选用法而被选择,并包装在...
每行对应一个唯一的单词,每列对应一个“概念”,即Wikipedia文章,并且每个条目都是文章j中单词i的TF-IDF分数。 矩阵保存在单独的块中,以节省内存。 medium_wiki.xml可以用作示例文件,以进行演示/测试,因为它...
大数据_项目_3 Tony Zheng 和我的大数据项目 3 的源代码 • 问题 给定多个文档,使用 MapReduce 计算单词语义相似度 ...计算词频 – 每个词的逆文档频率 (TF-IDF) 计算术语相似度 对术语相似度进行排序
值得一提是软件不仅可以统计出这些数据,还可以将这些数据按出现次数排序输出为Excel表格或Word表格文档。软件界面美观简洁、简单全面、实用方便,无需培训,即可快速上手,轻轻松松完成日常词频统计功能,真正做到...