techcrunch.git

@@ -1,4 +1,4 @@

-#!/usr/bin/python2.5

+#!/usr/bin/python

 # chmod 755 me, and make sure I have UNIX style newlines.

 #

 # techcrunch.py

@@ -10,10 +10,11 @@

 # TODO:

 # 1. Deep links: '<a class="post_more_link snap_nopreview" href="http://www.crunchgear.com/2010/08/21/fanboyism-when-expression-meets-desperation/">Read the rest of this entry &raquo;</a>'

 #   link = "http://techcrunch.com/2010/08/21/fanboyism-when-expression-meets-desperation/"

+# 2. Add Reddit counts: curl "http://www.reddit.com/api/info.json?url=http://i.imgur.com/HG9dJ.jpg"

 #

 # This file was coverted from tabs to spaces with the vim command %retab

 #

-# cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; mv techcrunch.yaml_back techcrunch.yaml

+# cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; cp techcrunch.yaml techcrunch_tmp.yaml; cp techcrunch.yaml_back techcrunch.yaml

 #

 import feedparser

@@ -35,12 +36,13 @@ import glob

 import smtplib

 import bisect

 import analysis

-import simplejson as json

+import json

 import cookielib

 import xml

 import texttime

 import operator

 from datetime import timedelta

+import cgi

 debug = True

 any_entry_added = False

@@ -78,7 +80,7 @@ html_footer = """

 </table>

 </div><br />

 <div align='center'>Thanks to <a href="http://www.feedparser.org/">The Universal Feed Parser module</a>,

-<a href="http://pyyaml.org/">PyYAML</a> and <a href="http://code.google.com/apis/chart/">Google Charts</a>.<br /><a href="techcrunch.yaml">raw data</a> &bull; <a href="stats.txt">status</a></div><br />

+<a href="http://pyyaml.org/">PyYAML</a> and <a href="http://code.google.com/apis/chart/">Google Charts</a>.<br /><a href="techcrunch.yaml">raw data</a> &bull; <a href="stats.txt">status</a><br />&copy; 2011 <a href="http://david.dlma.com">David Blume</a></div><br />

 </BODY>

 </HTML>

 """

@@ -109,6 +111,7 @@ def asciiize( s ):

 def sendEmail( subject, message, toaddrs, fromaddr='"techcrunch.py" <techcrunch@techcrunch.dlma.com>' ):

     """Sends Email"""

     smtp = smtplib.SMTP( 'localhost' )

+    smtp.login( user, passw )

     smtp.sendmail( fromaddr, \

                    toaddrs, \

                    "Content-Type: text/plain; charset=\"us-ascii\"\r\nFrom: %s\r\nTo: %s\r\nSubject: %s\r\n%s" % \

@@ -248,10 +251,8 @@ def process_feed( yaml_items ):

                     print "the feed returned feed.status %d." % ( feed.status, )

             else:

                 # Save off this

-                if hasattr( feed, 'bozo_exception' ) and type( feed.bozo_exception ) == xml.sax._exceptions.SAXParseException:

-                    print "Didn't pickle because of bozo_exception %s." % ( str( feed.bozo_exception ) )

-                elif hasattr( feed, 'bozo_exception' ) and isinstance( feed.bozo_exception, xml.sax._exceptions.SAXParseException ):

-                    print "Didn't pickle because of bozo_exception instance %s." % ( str( feed.bozo_exception ) )

+                if hasattr( feed, 'bozo_exception' ) and isinstance( feed.bozo_exception, xml.sax._exceptions.SAXParseException ):

+                    print "Didn't pickle TechCrunch feed because it had a bozo_exception: %s" % ( str( feed.bozo_exception ) )

                 else:

                     f = file( os.path.join( localdir, 'techcrunch_feed.pickle' ), 'wb' )

                     try:

@@ -271,12 +272,12 @@ def process_feed( yaml_items ):

             while len( yaml_items ) > 200:

                 yaml_items.pop()

-            cookie = Get_cookie( urllib2.Request( 'http://mediacdn.disqus.com/1078/build/system/count.js' ) )

+#            cookie = Get_cookie( urllib2.Request( 'http://mediacdn.disqus.com/1078/build/system/count.js' ) )

             for i in yaml_items:

                 # i['title'] = asciiize( i['title'] )

                 # i['tags'] = map( asciiize, i['tags'] )

-                process_yaml_item( i, cookie )

+                process_yaml_item( i )

     else:

         if hasattr(feed, 'bozo_exception'):

@@ -299,10 +300,7 @@ def process_feed( yaml_items ):

                 if print_last_line:

                     print "the feed had a URLError %s" % ( str(e), )

             elif isinstance( e, httplib.BadStatusLine ):

-                if hasattr(e, 'message'):

-                    print "the feed gave a bad status line %s." % ( str(e.message ), )

-                else:

-                    print "the feed gave a bad status line."

+                print "the feed gave a bad status line. (%s)" % ( str(e), )

             else:

                 if len( str(e) ):

                     print "the feed bozo_exception: %s \"%s\"" % ( str(e.__class__), str(e) )

@@ -352,6 +350,7 @@ def process_item( feed_item, yaml_items ):

                       'qualified'           : -1,

                       'comment_times'       : [],

                       'comments'            : [],

+                      'fb_shares'           : [],

                       'slash_comment_times' : [],

                       'slash_comments'      : [],

                       'retweet_times'       : [],

@@ -374,17 +373,19 @@ def process_item( feed_item, yaml_items ):

         yaml_item['slash_comment_times'].append( timecode_now )

         yaml_item['slash_comments'].append( int( feed_item.slash_comments ) )

-def process_yaml_item( yaml_item, cookie ):

+def process_yaml_item( yaml_item ):

     global any_entry_added

     timecode_now = int( time.time() )

     if len( yaml_item['comments'] ) < 8:

-        disqus_id = Get_disqus_id( yaml_item )

-        num_comments = Get_num_disqus_comments( yaml_item['link'], disqus_id, cookie )

+        num_shares, num_comments = Get_fb_stats( yaml_item['link'] )

+#        disqus_id = Get_disqus_id( yaml_item )

+#        num_comments = Get_num_disqus_comments( yaml_item['link'], disqus_id, cookie )

         if num_comments != -1:

             any_entry_added = True

             yaml_item['comment_times'].append( timecode_now )

             yaml_item['comments'].append( num_comments )

+            yaml_item['fb_shares'].append( num_shares )

     if len( yaml_item['retweets'] ) < 8:

         num_retweets = Get_num_retweets( yaml_item )

@@ -522,14 +523,17 @@ def Get_num_retweets_unused( yaml_item ):

 def Get_num_retweets( yaml_item ):

     url_string = yaml_item['link']

     try:

-        f = urllib2.urlopen( 'http://urls.api.twitter.com/1/urls/count.json?url=%s&callback=twttr.receiveCount' % urllib.quote_plus( url_string ) )

+        f = urllib2.urlopen( 'http://urls.api.twitter.com/1/urls/count.json?url=%s&callback=twttr.receiveCount' % \

+                             urllib.quote_plus( url_string ) )

         data = f.read()

         f.close()

-    except urllib2.URLError, e:

+    except (urllib2.URLError, httplib.BadStatusLine), e:

         if hasattr( e, 'reason' ):

             print "Get_num_retweets got an error:", e.reason

         elif hasattr( e, 'code' ):

             print "Get_num_retweets got an error. Code:", e.code

+        else:

+            print "Get_num_retweets got an error:", str( e )

         return -1

     tag_to_find = '"count":'

     offset = data.find( tag_to_find )

@@ -549,6 +553,35 @@ def Get_num_retweets( yaml_item ):

                           )

     return -1

+def Get_fb_stats( url_string ):

+    """ Returns shares and comments """

+    shares = -1

+    comments = -1

+    try:

+        f = urllib2.urlopen( 'https://graph.facebook.com/?ids=' + url_string )

+        data = f.read()

+        f.close()

+    except (urllib2.URLError, httplib.BadStatusLine), e:

+        if hasattr( e, 'reason' ): # URLError

+            print "Get_fb_stats got an error:", e.reason, url_string

+        elif hasattr( e, 'code' ): #URLError

+            print "Get_fb_stats got an error. Code:", e.code, url_string

+        else:

+            print "Get_fb_stats got an error:", str( e )

+        return -1, -1

+    if len( data ) > len( url_string ):

+        d = json.loads( data ).values()[0]

+        if d.has_key( 'shares' ):

+            shares = d['shares']

+        else:

+            shares = 0

+        if d.has_key( 'comments' ):

+            comments = d['comments']

+        else:

+            comments = 0

+    else:

+        print "Get_fb_stats got too little data for ",  url_string

+    return shares, comments

 def Save_image( url_string, file_path ):

@@ -556,12 +589,15 @@ def Save_image( url_string, file_path ):

         f = urllib2.urlopen( url_string )

         data = f.read()

         f.close()

-    except urllib2.URLError, e:

-        if hasattr( e, 'reason' ):

-            print "Save_image got an error:", e.reason

-        elif hasattr( e, 'code' ):

-            print "Save_image got an error. Code:", e.code

+    except (urllib2.URLError, httplib.BadStatusLine), e:

+        if hasattr( e, 'reason' ): # URLError

+            print "Save_image got an error attempting to create", file_path, "Reason:", e.reason

+        elif hasattr( e, 'code' ): # URLError

+            print "Save_image got an error attempting to create", file_path, "Code:", e.code

+        else:

+            print "Save_image got an error from urlopen", e

         return url_string

+

     if len( data ) > 50:

         f = open( file_path, 'wb' )

         f.write( data )

@@ -575,7 +611,7 @@ def Make_index_html( yaml_items, weekend_stats, weekday_stats ):

     index_fullpath = os.path.join( localdir, 'index.html' )

     cache_path = os.path.join( localdir, 'cache' )

-    files_to_delete = glob.glob( cache_path + '*.png' )

+    files_to_delete = glob.glob( os.path.join( cache_path, '*.png' ) )

     f = file( new_index_fullpath, 'w' )

     f.write( html_head % ( even_background, odd_background ) )

@@ -588,41 +624,39 @@ def Make_index_html( yaml_items, weekend_stats, weekday_stats ):

     f.write( '<div align="center">\n<table>\n' )

-    image_index = 0

-    for i in yaml_items[:40]:

+    for image_index, image in enumerate(yaml_items[:40]):

         tag_hit = False

-        if i['author'].lower() in authors_to_post:

+        if image['author'].lower() in authors_to_post:

             tag_hit = True

-        elif len( set([j.lower() for j in i['tags']]) & tags_to_post ) > 0:

+        elif len( set([j.lower() for j in image['tags']]) & tags_to_post ) > 0:

             tag_hit = True

-        chart_url = make_chart_url( i['orig_posted'],

-                                    i['comment_times'],

-                                    i['comments'],

-                                    i['retweet_times'],

-                                    i['retweets'],

-                                    i['qualified'],

+        chart_url = make_chart_url( image['orig_posted'],

+                                    image['comment_times'],

+                                    image['comments'],

+                                    image['retweet_times'],

+                                    image['retweets'],

+                                    image['qualified'],

                                     image_index % 2,

                                     tag_hit

                                   )

-#        if i['title'].startswith( 'Too ' ):

-#            print i['title'], i['qualified'], i['retweet_times']

+#        if image['title'].startswith( 'Too ' ):

+#            print image['title'], image['qualified'], image['retweet_times']

 #            print chart_url

         image_url = Save_image( chart_url, os.path.join( cache_path, '%d_%d.png' % ( cur_time, image_index ) ) )

         f.write( '<tr valign="center" class="%s">\n  <td><strong><a href="%s">%s</a></strong> <span class="author">by %s</span></td>\n' % \

                  ( image_index % 2 and "even" or "odd",

-                   i['link'],

-                   i['title'].encode( 'ascii', 'xmlcharrefreplace' ),

-                   i['author'].encode( 'ascii', 'xmlcharrefreplace' ),

+                   image['link'],

+                   image['title'].encode( 'ascii', 'xmlcharrefreplace' ),

+                   image['author'].encode( 'ascii', 'xmlcharrefreplace' ),

                  )

                )

-        f.write( '  <td>%s<td>\n' % ( i['qualified'] != -1 and '<img src="star_30.png" width="30" height="29" />' or '' ) )

+        f.write( '  <td>%s<td>\n' % ( image['qualified'] != -1 and '<img src="star_30.png" width="30" height="29" />' or '' ) )

         f.write( '  <td><img src="%s" width="%d" height="%d" border="0" /></td></tr>\n' % \

                  ( image_url,

                    img_width,

                    img_height

                  )

                )

-        image_index += 1

     f.write( html_footer )

     f.close()

     if os.path.exists( index_fullpath ):

@@ -634,13 +668,15 @@ def Make_index_html( yaml_items, weekend_stats, weekday_stats ):

 def Make_feed_file( yaml_items ):

     f = open( os.path.join( localdir, 'rss_feed.xml' ), 'wb' )

     f.write( "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n<rss version=\"2.0\">\n<channel>\n<title>Trending at TechCrunch</title><link>http://techcrunch.dlma.com</link>" )

-    f.write( "<pubDate>%s</pubDate><description>Automatically Generated Feed</description><language>en-us</language>" % ( time.strftime( "%a, %d %b %Y %H:%M:%S +0000", time.gmtime() ) ) )

+    f.write( "<pubDate>%s</pubDate><description>Automatically Generated Feed</description><language>en-us</language>\n" % ( time.strftime( "%a, %d %b %Y %H:%M:%S +0000", time.gmtime() ) ) )

     count = 0

     for item in yaml_items:

         now = time.strftime( "%a, %d %b %Y %H:%M:%S +0000", time.gmtime( item['orig_posted'] ) )

         if item['qualified'] != -1:

+            escaped_title = cgi.escape( item['title'] ).encode( 'ascii', 'xmlcharrefreplace' )

+            escaped_author = cgi.escape( item['author'] ).encode( 'ascii', 'xmlcharrefreplace' )

             f.write( "<item><title>%s</title><pubDate>%s</pubDate><link>%s</link><guid isPermaLink=\"false\">%s</guid><description><![CDATA[By: %s]]></description></item>\n" % \

-                     ( item['title'].encode( 'ascii', 'xmlcharrefreplace' ), now, item['link'], item['link'], item['author'].encode( 'ascii', 'xmlcharrefreplace' ) ) )

+                     ( escaped_title, now, item['link'], item['link'], escaped_author ) )

             count += 1

             if count > 14:

                 break

@@ -668,10 +704,9 @@ if __name__=='__main__':

         #     'qualified'           : -1

         #     'comment_times'       : [ 1282197199, 1282197407 ]

         #     'comments'            : [ 0, 15 ]

+        #     'fb_shares'           : [ 0, 3 ]

         #     'slash_comment_times' : [ 1282197199, 1282197407 ]

         #     'slash_comments'      : [ 0, 5 ]

-        #     'slash_comment_times' : [ 1282197199, 1282197407 ]

-        #     'slash_comments'      : [ 0, 3 ]

         #     'retweet_times'       : [ 1282197199, 1282197407 ]

         #     'retweets'            : [ 0, 43 ]

         #    },

@@ -682,6 +717,11 @@ if __name__=='__main__':

         if os.path.exists( yaml_fullpath ):

             f = file( yaml_fullpath, 'rb' )

             items = yaml.load( f )

+

+            # Do any dictionary item updating that might be necessary

+#            for item in items:

+#                if not item.has_key( 'fb_shares' ):

+#                    item['fb_shares'] = []

             f.close()

         else:

             print "could not open", yaml_fullpath

@@ -754,7 +794,7 @@ if __name__=='__main__':

         traceback.print_exc( file = sys.stdout )

         try:

             sendEmail( 'Exception thrown in techcrunch.py',

-                       exceptional_text,

+                       exceptional_text + "\n" + traceback.format_exc(),

                        ( 'david.blume@gmail.com', ) )

         except Exception, e:

             print "Could not send email to notify you of the exception. :("

@@ -777,7 +817,7 @@ if __name__=='__main__':

     lines = lines[:168] # Just keep the past week's worth

     # status = len( message.strip() ) and message.strip().replace( '\n', ' - ' ) or "OK"

     status = len( message.strip() ) and '\n                       '.join( message.splitlines() ) or "OK"

-    lines.insert( 0, "%s %3.0fs %s\n" % ( time.strftime('%H:%M, %Y-%m-%d', time.localtime()), time.time() - start_time, status ))

+    lines.insert( 0, "%s %3.0fs %s\n" % ( time.strftime('%Y-%m-%d, %H:%M', time.localtime()), time.time() - start_time, status ))

     f = open( os.path.join( localdir,'stats.txt' ), 'w' )

     f.writelines( lines )

     f.close()