implement idf

Makoto Hiramatsu · Makoto Hiramatsu · commit 8ae8ee42547c · 2015-02-26T01:43:16.000+09:00
diff --git a/lib/r_nlp.rb b/lib/r_nlp.rb
@@ -1,5 +1,6 @@
 require "r_nlp/version"
 require "r_nlp/tf"
+require 'r_nlp/idf'
 
 module RNlp
   # Your code goes here...
diff --git a/lib/r_nlp/idf.rb b/lib/r_nlp/idf.rb
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+module RNlp
+  class Idf
+    # compatible with ja or en
+    attr_reader :lang
+    def initialize(lang)
+      @lang = lang
+      unless lang == 'ja' || lang == 'en'
+        puts "#{@lang} is not compatible language\nlang should be 'ja' or 'en'"
+        exit
+      end
+    end
+    # documents should be array of string
+    def calc_idf(word, documents)
+      @word = word
+      @documents = documents
+      n = @documents.size
+      df = 0.0
+      @documents.each do |document|
+        df += 1 if document =~ /#{@word}/
+      end
+      idf = Math.log2(n/df) + 1
+      return idf
+    end
+  end
+end
diff --git a/lib/r_nlp/tf.rb b/lib/r_nlp/tf.rb
@@ -7,6 +7,10 @@ class Tf
     attr_reader :lang
     def initialize(lang)
       @lang = lang
+      unless lang == 'ja' || lang == 'en'
+        puts "lang #{@lang} is not compatible."
+        exit
+      end
     end
     def count(text)
       tf = Hash.new
@@ -23,7 +27,7 @@ def count(text)
           end
         end
       elsif @lang == 'en'
-        text.split(" ").each do |line|
+        text.split("\n").each do |line|
           line.split(" ").each do |word|
             if tf[word] == nil
               tf[word] = 1
@@ -32,9 +36,6 @@ def count(text)
             end
           end
         end
-      else
-        puts "lang #{@lang} is not compatible."
-        exit
       end
       return tf
     end
diff --git a/lib/r_nlp/version.rb b/lib/r_nlp/version.rb
@@ -1,3 +1,3 @@
 module RNlp
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end
