NLP_Working/MeCab.cpp at master · Shusei-E/NLP_Working · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
// Reference: http://d.hatena.ne.jp/syou6162/20090410/1239289505
#include <mecab.h>
#include <map>
#include <regex>

typedef map<string, int>::const_iterator map_freq_it; //下のみたいに毎回型を書いているのが面倒なときは、typedefで名前をつける
typedef std::vector<map_freq_it>::const_iterator vec_stu_citer_t;

// Defined in another place
//vector<string> split( string s, string c ){
//  vector<string> ret;
//  for( int i=0, n; i <= s.length(); i=n+1 ){
//	n = s.find_first_of( c, i );
//	if( n == string::npos ) n = s.length();
//	string tmp = s.substr( i, n-i );
//	ret.push_back(tmp);
//  }
//  return ret;
//}

bool compare( const map_freq_it& a, const map_freq_it& b ){ // ファンクタ
  return ( a->second > b->second );
}

void DoMeCab(DATA_USE *Data, int used_row_num, string path, string file_name){

	map<string, int> freq; // store_frequency
	map<string, int>::iterator find_it;
	map<string, int>::iterator output_it;

	file_name.erase(file_name.end()-9, file_name.end());
	string save_path = path + file_name + "Freq.txt";
	string save_summary_path = path + "Summary.txt";
	cout << "Save as: " << save_path << endl;


  cout << "The number of rows used: " << used_row_num << endl;

	MeCab::Tagger *tagger = MeCab::createTagger("");
	for(int row_num=0; row_num< used_row_num; ++row_num){


    const MeCab::Node* node = tagger->parseToNode(Data[row_num].Text.c_str());


    for (; node; node = node->next) {

			vector<string> resvec = split( node->feature, "," );

			if(resvec[0]=="名詞" && resvec[6]!="*" ){

				/* When it is a noun */
				string noun = resvec[6];
				find_it = freq.find(noun);
				if(resvec[1]=="数"|resvec[1]=="非自立"|resvec[1]=="代名詞"|resvec[1]=="接尾") continue;

				if (find_it != freq.end()){
					find_it->second += 1;
				}else{
					freq.insert(pair<string, int>(noun, 1));
				}

			}//close if for finding re_feature


   	}//close for node

	}//close for Data analyzing (analyze all rows)

		/* Make an output */
		vector<map_freq_it> sorted;
 		for(map_freq_it mfi = freq.begin(); mfi != freq.end(); ++mfi)
			sorted.push_back(mfi);

		sort(sorted.begin(), sorted.end(), compare);

		int output_number = 0; int max_output = 500; // How many top words you want to get

		//// Save Each file
		//std::ofstream ofs(save_path, std::ios::out | std::ios::app );
 	 	//for(vec_stu_citer_t output_it = sorted.begin(); output_it != sorted.end(); ++output_it){
		//	ofs << (*output_it)->first << "," <<  (*output_it)->second << endl;
 	 	//
		//	if(output_number > max_output) break;
		//	++output_number;
 		//}

		// Save Summary File
		vector<string> datetime_vec = split(Data[0].Datetime, " " );
		char station[1024];
		 strncpy(station, Data[0].Station.c_str(), sizeof(station));
		 station[sizeof(station) - 1] = 0;
		output_number = 0;

		std::ofstream ofs2(save_summary_path, std::ios::out | std::ios::app );
		for(vec_stu_citer_t output_it = sorted.begin(); output_it != sorted.end(); ++output_it){
			ofs2 << datetime_vec[0] << "," << (*output_it)->first << "," <<  (*output_it)->second << "," << station << endl;

			if(output_number > max_output) break;
			++output_number;
 		}

		delete tagger;
}//close function