idf counting replaced to crawl_acm

crawl_acm

@@ -3,6 +3,7 @@
 import search
 c = search.Crawler('db_acm')
 c.create_index_tables()
-c.crawl("http://dl.acm.org/pub.cfm?id=J204&CFID=261281623&CFTOKEN=70296603") 
+c.crawl("http://dl.acm.org/pub.cfm?id=J204&CFID=261281623&CFTOKEN=70296603")
+c.count_idf()
 
 

search.py

@@ -13,6 +13,7 @@
 import urllib2
 from BeautifulSoup import *
 from pysqlite2 import dbapi2 as sqlite
+import math
 
 
 
@@ -215,6 +216,22 @@ def crawl(self, journal_url, depth=2):
         print "%4d papers were indexed" % count
 
 
+    def count_idf(self):
+        print "Counting idf..."
+        url_count = self.con.execute("select count(rowid) from url_list").fetchone()[0]
+        words_count = self.con.execute("select count(rowid) from word_list").fetchone()[0]
+        max = 0
+        for word_id in range(1, words_count + 1):
+            urls = self.con.execute("select distinct url_id from word_location "
+                                        "where word_id = %s" % word_id).fetchall()
+            num = len(urls)
+            if num > max:
+                   max = num
+            idf = math.log10(url_count / num)
+            self.con.execute("update word_list set idf = %f where rowid = %d" % (idf, word_id))
+        self.db_commit()
+
+
     def create_index_tables(self):
         """ Create database tables """
         res = self.con.execute('select name from sqlite_master where type="table" and name="url_list"').fetchone()

searcher.py

@@ -46,21 +46,6 @@ def get_top_words(self, words, n):
         else:
             return sorted_top[0: n]
 
-    def count_idf(self):
-        print "Counting idf..."
-        url_count = self.con.execute("select count(rowid) from url_list").fetchone()[0]
-        words_count = self.con.execute("select count(rowid) from word_list").fetchone()[0]
-        max = 0
-        for word_id in range(1, words_count + 1):
-            urls = self.con.execute("select distinct url_id from word_location "
-                                        "where word_id = %s" % word_id).fetchall()
-            num = len(urls)
-            if num > max:
-                   max = num
-            idf = math.log10(url_count / num)
-            self.con.execute("update word_list set idf = %f where rowid = %d" % (idf, word_id))
-        self.db_commit()
-
 
     def tf(self, words):
         '''Return top n words in text'''

start_search

@@ -2,7 +2,6 @@
 
 import searcher
 s = searcher.Searcher('db_acm')
-s.count_idf()
 s.cos_search("Quality of service (QoS) can be a critical element for achieving the business goals of a service provider, for the acceptance of a service by the user, or for guaranteeing service characteristics in a composition of services, where a service is defined as either a software or a software-support (i.e., infrastructural) service which is available on any type of network or electronic channel. The goal of this article is to compare the approaches to QoS description in the literature, where several models and metamodels are included. consider a large spectrum of models and metamodels to describe service quality, ranging from ontological approaches to define quality measures, metrics, and dimensions, to metamodels enabling the specification of quality-based service requirements and capabilities as well as of SLAs (Service-Level Agreements) and SLA templates for service provisioning.", 10)