techcrunch.git

@@ -312,7 +312,8 @@ def process_yaml_item( yaml_item, cookie ):

     timecode_now = int( time.time() )

     if len( yaml_item['comments'] ) < 8:

-        num_comments = Get_num_disqus_comments( yaml_item['link'], cookie )

+        disqus_id = Get_disqus_id( yaml_item )

+        num_comments = Get_num_disqus_comments( yaml_item['link'], disqus_id, cookie )

         if num_comments != -1:

             any_entry_added = True

             yaml_item['comment_times'].append( timecode_now )

@@ -349,35 +350,35 @@ def Get_num_comments( url_string ):

 def Get_cookie( cookie_request ):

     cookie = cookielib.CookieJar()

+    error_string = "Get_cookie didn't."

     try:

         cookie_response = urllib2.urlopen( cookie_request )

         cookie.extract_cookies( cookie_response, cookie_request )

         return cookie

     except urllib2.URLError, e:

         if hasattr( e, 'reason' ):

-            print "Get_cookie got an error:", e.reason

+            error_string = "Get_cookie got an error: %s" % ( str( e.reason ) )

     elif hasattr( e, 'code' ):

-            print "Get_cookie got an error. Code:", e.code

+            error_string = "Get_cookie got an error. Code: %s" % ( str( e.code ) )

+    print error_string

     return None

-def Get_num_disqus_comments( url_string, cookie ):

-

-    if cookie == None:

-        return -1

-

+def Get_disqus_id( yaml_item ):

+    if 'disqus_id' in yaml_item:

+        return yaml_item['disqus_id']

+    url_get_data = ''

     try:

-        f = urllib2.urlopen( url_string )

+        f = urllib2.urlopen( yaml_item['link'] )

         data = f.read()

         f.close()

     except urllib2.URLError, e:

         if hasattr( e, 'reason' ):

-            print "Get_num_disqus_comments got an error:", e.reason

+            print "Get_disqus_id got an error:", e.reason

     elif hasattr( e, 'code' ):

-            print "Get_num_disqus_comments got an error. Code:", e.code

-        return -1

+            print "Get_disqus_id got an error. Code:", e.code

+        return url_get_data

     tag_to_find = '<a href="#comments" rel="nofollow"><span class="dsq-postid" rel="'

-    disqus_tag_to_find = 'displayCount('

     offset = data.find( tag_to_find )

     if offset != -1:

         start_pos = offset + len( tag_to_find )

@@ -385,9 +386,19 @@ def Get_num_disqus_comments( url_string, cookie ):

         while data[ end_pos ] != '"' and end_pos < start_pos + 200:

             end_pos += 1

         if end_pos < start_pos + 200:

+        url_get_data = urllib.quote_plus( data[start_pos:end_pos] ).replace( '+', '%20' )

+        yaml_item['disqus_id'] = url_get_data

+#    else:

+#        print "Get_disqus_id could not find #comments anchor for", yaml_item['link']

+    return url_get_data

+

+def Get_num_disqus_comments( url_string, disqus_id, cookie ):

+

+    if cookie == None or disqus_id == '':

+        return -1

+

     opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cookie ) )

-            url_GET_data = urllib.quote_plus( data[start_pos:end_pos] ).replace( '+', '%20' )

-            request = urllib2.Request( 'http://disqus.com/forums/techcrunch/count.js?q=1&0=1,' + url_GET_data )

+    request = urllib2.Request( 'http://disqus.com/forums/techcrunch/count.js?q=1&0=1,' + disqus_id )

     try:

         response = opener.open( request )

         disqus_data = response.read()

@@ -396,14 +407,15 @@ def Get_num_disqus_comments( url_string, cookie ):

             print "Get_num_disqus_comments got an error getting the count:", e.reason

         elif hasattr( e, 'code' ):

             print "Get_num_disqus_comments got an error getting the count. Code:", e.code

-        disqus_data = ""

+    disqus_tag_to_find = 'displayCount('

     disqus_offset = disqus_data.find( disqus_tag_to_find )

     if disqus_offset != -1:

         start_pos = disqus_offset + len( disqus_tag_to_find )

         end_pos = disqus_data.find( '}]})', start_pos )

         if end_pos != -1:

             return int( json.loads( disqus_data[start_pos:end_pos+3] )['counts'][0]['comments'] )

-

+    else:

+        print "Get_num_disqus_comments found no disqus tag for", url_string

     return -1

 def Get_num_retweets( url_string ):