techcrunch.git

@@ -7,7 +7,11 @@ and decide which articles to include in its own feed.

 Here's a [blog post about it](http://david.dlma.com/blog/my-techcrunch-feed-filter).

-# History

+# To Do

+

+* Maybe use Reddit upvotes

+

+# Pre-Git History

 This was originally archived in a Subversion repo. I'd forgotten about the

 version control and had gotten into the habit of just modifying the production

@@ -1,13 +1,8 @@

 #!/usr/bin/env python

 #

-# TODO:

-# 1. Deep links: '<a class="post_more_link snap_nopreview" href="http://www.crunchgear.com/2010/08/21/fanboyism-when-expression-meets-desperation/">Read the rest of this entry &raquo;</a>'

-#   link = "http://techcrunch.com/2010/08/21/fanboyism-when-expression-meets-desperation/"

-# 2. Add Reddit counts: curl "http://www.reddit.com/api/info.json?url=http://i.imgur.com/HG9dJ.jpg"

-#

-# cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; cp techcrunch.yaml techcrunch_tmp.yaml; cp techcrunch.yaml_back techcrunch.yaml

-#

-

+# Testing without affecting the yaml file and saving the updated one aside:

+# cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; \

+# cp techcrunch.yaml techcrunch_tmp.yaml; cp techcrunch.yaml_back techcrunch.yaml

 import feedparser

 import yaml

 import sys

@@ -32,6 +27,7 @@ import texttime

 import operator

 from datetime import timedelta

 import cgi

+import smtp_creds  # Your own credentials, used in send_email()

 debug = True

 any_entry_added = False

@@ -102,10 +98,11 @@ def asciiize(s):

         return s

-def sendEmail(subject, message, toaddrs, fromaddr='"techcrunch.py" <techcrunch@techcrunch.dlma.com>'):

+def send_email(subject, message, toaddrs,

+        fromaddr='"%s" <%s>' % (os.path.basename(__file__), smtp_creds.user)):

     """ Sends Email """

-    smtp = smtplib.SMTP('mail.dlma.com', port=587)

-    smtp.login(user, passw)

+    smtp = smtplib.SMTP(smtp_creds.server, port=smtp_creds.port)

+    smtp.login(smtp_creds.user, smtp_creds.passw)

     smtp.sendmail(fromaddr, \

                   toaddrs, \

                   "Content-Type: text/plain; charset=\"us-ascii\"\r\nFrom: %s\r\nTo: %s\r\nSubject: %s\r\n%s" % \

@@ -125,8 +122,6 @@ def make_chart_url(time_posted, lhs_times, lhs_values, rhs_times,

 #    lhs_times, lhs_values = zip(*comments)

 #    rhs_times, rhs_values = zip(*rhs)

-    # TODO handle failure cases, -1

-

     if not len(lhs_times):

         lhs_times = [time_posted,]

     if not len(lhs_values):

@@ -136,8 +131,6 @@ def make_chart_url(time_posted, lhs_times, lhs_values, rhs_times,

     if not len(rhs_values):

         rhs_values = [0,]

-#    lhs_times = [(i - time_posted + 900) / 1800 for i in lhs_times]

-#    rhs_times = [(i - time_posted + 900) / 1800 for i in rhs_times]

     lhs_times = [(i - time_posted) / 1800 for i in lhs_times]

     rhs_times = [(i - time_posted) / 1800 for i in rhs_times]

@@ -212,11 +205,9 @@ def make_chart_url(time_posted, lhs_times, lhs_values, rhs_times,

 def process_feed(yaml_items):

-    """

-    Retrieve the url and process it.

+    """Retrieve the url and process it.

     feed_info (in, out) A tuple that describes an individual feed, like its name and etag.

     """

-

     feed = feedparser.parse('http://feeds.feedburner.com/TechCrunch')

     if hasattr(feed, 'status'):

         if feed.status == 304:

@@ -399,22 +390,7 @@ def Get_reddit_stats(url_string):

 def Get_fb_stats(url_string):

-    """ There are apparently two pretty good ways to do this. One, with FQL, querying for the parameters you want,

-    and two, with URL id. They go like this:

-

-    FQL:

-

-    u = urllib.quote_plus(url_string)

-    urllib2.urlopen('https://graph.facebook.com/fql?q=SELECT%%20total_count,comment_count,like_count,share_count%%20FROM%%20link_stat%%20WHERE%%20url=%%27%s%%27' % (u))

-

-    URL ID:

-

-    u = urllib.quote_plus(url_string)

-    with open('facebook-token.txt', 'r') as f:

-        token = f.read()

-    encoded = urllib.urlencode({'access_token': token})

-    urllib2.urlopen('https://graph.facebook.com/vX.Y/?id=%s&%s' % (u, encoded)

-    """

+    """Use graph's "engagement" field to get reactions and shares."""

     shares = -1

     comments = -1

     likes = -1

@@ -423,21 +399,7 @@ def Get_fb_stats(url_string):

     try:

         encoded = urllib.urlencode({'access_token': facebook_token})

-#        url = 'https://graph.facebook.com/fql?q=SELECT%%20total_count,comment_count,like_count,share_count%%20FROM%%20link_stat%%20WHERE%%20url=%%27%s%%27&%s'

-        # This stopped working 2018-01-13, 11:25, after I told Facebook the app would use v2.11

-        # https://developers.facebook.com/docs/graph-api/changelog/version2.9#gapi-deprecate

-        # url = 'https://graph.facebook.com/v2.8/?id=%s&fields=og_object{engagement},share&%s'

-

-        # Consider the following for a different engagement field:

-        #   "engagement": {

-        #     "reaction_count": 115,

-        #     "comment_count": 0,

-        #     "share_count": 102,

-        #     "comment_plugin_count": 0

-        #   },

-        # Where reaction_count + share_count = og_object.engagement.count

         url = 'https://graph.facebook.com/v2.11/?id=%s&fields=engagement&%s'

-

         f = urllib2.urlopen(url % (urllib.quote_plus(url_string), encoded))

         data = f.read()

         f.close()

@@ -486,18 +448,18 @@ def Get_fb_stats(url_string):

     return shares, comments, likes

-def Save_image(url_string, file_path):

+def save_image(url_string, file_path):

     try:

         f = urllib2.urlopen(url_string)

         data = f.read()

         f.close()

     except (urllib2.URLError, httplib.BadStatusLine) as e:

         if hasattr(e, 'reason'): # URLError

-            print "Save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Reason:", e.reason

+            print "save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Reason:", e.reason

         elif hasattr(e, 'code'): # URLError

-            print "Save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Code:", e.code

+            print "save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Code:", e.code

         else:

-            print "Save_image: Error from urlopen", e

+            print "save_image: Error from urlopen", e

         return url_string

     if len(data) > 50:

@@ -507,7 +469,8 @@ def Save_image(url_string, file_path):

     return url_string

-def Make_index_html(yaml_items, weekend_stats, weekday_stats):

+def make_index_html(yaml_items, weekend_stats, weekday_stats):

+    """Writes a static index.html file from the YAML items."""

     cur_time = int(time.time())

     new_index_fullpath = os.path.join(localdir, 'index.html_new')

     index_fullpath = os.path.join(localdir, 'index.html')

@@ -541,10 +502,7 @@ def Make_index_html(yaml_items, weekend_stats, weekday_stats):

                                        image_index % 2,

                                        tag_hit

                                       )

-#        if image['title'].startswith( 'Too ' ):

-#            print image['title'], image['qualified'], image['rhs_times']

-#            print chart_url

-            image_url = Save_image(chart_url, os.path.join(cache_path, '%d_%d.png' % (cur_time, image_index)))

+            image_url = save_image(chart_url, os.path.join(cache_path, '%d_%d.png' % (cur_time, image_index)))

             f.write('<tr valign="center" class="%s">\n  <td><strong><a href="%s">%s</a></strong> <span class="author">by %s</span></td>\n' % \

                      (image_index % 2 and "even" or "odd",

                       image['link'],

@@ -568,7 +526,8 @@ def Make_index_html(yaml_items, weekend_stats, weekday_stats):

         os.unlink(fname)

-def Make_feed_file(yaml_items):

+def make_feed_file(yaml_items):

+    """Writes the RSS feed file with the YAML items."""

     with codecs.open(os.path.join(localdir, 'rss_feed.xml'), 'wb', 'utf-8') as f:

         f.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<rss version=\"2.0\">\n<channel>\n<title>Trending at TechCrunch</title><link>http://techcrunch.dlma.com</link>")

         f.write("<pubDate>%s</pubDate><description>Automatically Generated Feed</description><language>en-us</language>\n" % (time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())))

@@ -631,7 +590,7 @@ if __name__=='__main__':

             print "could not open", yaml_fullpath

             items = []

-        with open('facebook-token.txt', 'r') as f:

+        with open(os.path.join(localdir, 'facebook-token.txt'), 'r') as f:

             facebook_token = f.read()

         progress_text = ["read techcrunch.yaml"]

@@ -688,9 +647,9 @@ if __name__=='__main__':

             with codecs.open(os.path.join(localdir, 'techcrunch_unicode.yaml'), 'w', 'utf-8') as f:

                 yaml.dump(items, f, encoding='utf-8', width=120)

-            Make_feed_file(items)

+            make_feed_file(items)

-            Make_index_html(items, weekend_stats, weekday_stats)

+            make_index_html(items, weekend_stats, weekday_stats)

         else:

             print "No entries were added this time."

@@ -699,7 +658,7 @@ if __name__=='__main__':

         print exceptional_text, ' '.join(progress_text)

         traceback.print_exc(file=sys.stdout)

         try:

-            sendEmail('Exception thrown in techcrunch.py',

+            send_email('Exception thrown in techcrunch.py',

                       exceptional_text + "\n" + traceback.format_exc(),

                       ('david.blume@gmail.com',))

         except Exception as e: