techcrunch.py (1ec086e) - techcrunch.git

1ec086ea5630b67a0c5304e1fd446476a1c7f420

Catch up to production agai... David Blume authored 6 years ago	`1) #!/usr/bin/env python`
Original 2010-09-03 version David Blume authored 6 years ago	`2) #`
Moved SMTP credentials to s... David Blume authored 6 years ago	`3) # Testing without affecting the yaml file and saving the updated one aside: 4) # cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; \ 5) # cp techcrunch.yaml techcrunch_tmp.yaml; cp techcrunch.yaml_back techcrunch.yaml`
Original 2010-09-03 version David Blume authored 6 years ago	`6) import feedparser 7) import yaml 8) import sys 9) import os 10) import time 11) import StringIO 12) import codecs 13) import traceback 14) import calendar 15) import pickle 16) import exceptions 17) import urllib 18) import urllib2 19) import httplib 20) import shutil 21) import glob 22) import smtplib 23) import analysis`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`24) import json`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`25) import xml`
2011-02-04: Update to the c... David Blume authored 6 years ago	`26) import operator`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`27) import cgi`
Moved SMTP credentials to s... David Blume authored 6 years ago	`28) import smtp_creds # Your own credentials, used in send_email()`
Original 2010-09-03 version David Blume authored 6 years ago	`29) 30) debug = True 31) any_entry_added = False`
Better conformance to PEP-8... David Blume authored 6 years ago	`32) tags_to_post = {'apple', 'google'}`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`33) authors_to_post = ['michael arrington',] 34)`
Catch up to production agai... David Blume authored 6 years ago	`35) # TODO 2018-01-18: Maybe combine fb_likes with bf_shares or something... 36) rhs_metric = 'fb_likes'`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`37) rhs_metric_times = 'comment_times'`
Original 2010-09-03 version David Blume authored 6 years ago	38) 39) localdir = '' 40) 41) html_head = """ 42) <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'> 43) <HTML><HEAD> 44) <title>TechCrunch Feed Filter</title> 45) <!-- <link rel="alternate" type="application/rss+xml" title="RSS feed" href="http://techcrunch.dlma.com/rss_feed.xml" /> --> 46) <link rel="alternate" type="application/rss+xml" title="RSS feed" href="http://feeds.feedburner.com/TrendingAtTechcrunch" /> 47) <style type="text/css"> 48) body { font-family: "Arial", san-serif; } 49) .author { font-size: smaller; } 50) .h3 { font-size: larger; } 51) a { text-decoration: none; } 52) /* table { border: none; border-collapse:collapse; font-size: large } */ 53) table { border-collapse: collapse; } 54) table.legend { border:1px solid LightSlateGray; font-size: medium; border-collapse:separated; } 55) table.legend th { border: 1px solid LightSlateGray; background-color: #E0E0E0; } 56) table.legend td { border: 1px solid LightSlateGray; } 57) tr.even { background:#%s; padding: 2em; } 58) tr.odd { background:#%s; padding-bottom: 2em; } 59) </style> 60) </HEAD> 61) <BODY> 62) <div align='center'><h3>TechCrunch Feed Filter</h3></div>
Catch up to production agai... David Blume authored 6 years ago	`63) This page shows what analysis is done to filter the noise away from the Techcrunch feed into <a href="http://feeds.feedburner.com/TrendingAtTechcrunch">a more concise feed</a>. <a href="http://david.dlma.com/blog/my-techcrunch-feed-filter">Learn more about the Feed Filter</a>.<br /><br />`
Original 2010-09-03 version David Blume authored 6 years ago	`64) """ 65) 66) html_footer = """ 67) </table> 68) </div><br /> 69) <div align='center'>Thanks to <a href="http://www.feedparser.org/">The Universal Feed Parser module</a>,`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`70) <a href="http://pyyaml.org/">PyYAML</a> and <a href="http://code.google.com/apis/chart/">Google Charts</a>.<br /><a href="techcrunch.yaml">raw data</a> • <a href="stats.txt">status</a><br />© 2011 <a href="http://david.dlma.com">David Blume</a></div><br />`
Original 2010-09-03 version David Blume authored 6 years ago	`71) </BODY> 72) </HTML> 73) """ 74) 75) img_width = 300 76) img_height = 50 77) 78) series_1_color = "0000FF" 79) series_2_color = "00AA00" 80) threshold_color = "FF8C00"`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`81) tag_color = "F01000"`
Original 2010-09-03 version David Blume authored 6 years ago	`82) 83) even_background = "F8F8F8" 84) odd_background = "E8E8E8" 85)`
2011-02-04: Update to the c... David Blume authored 6 years ago	`86) even_watermark = "E0E0FF" 87) odd_watermark = "D0D0F0" 88) 89)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`90) def asciiize(s):`
Original 2010-09-03 version David Blume authored 6 years ago	`91) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`92) return s.encode('ascii')`
Better conformance to PEP-8... David Blume authored 6 years ago	`93) except UnicodeEncodeError:`
Original 2010-09-03 version David Blume authored 6 years ago	`94) return s`
Better conformance to PEP-8... David Blume authored 6 years ago	`95) except exceptions.AttributeError:`
Original 2010-09-03 version David Blume authored 6 years ago	`96) return s 97)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`98)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`99) def send_email(subject, message, toaddrs, 100) fromaddr='"%s" <%s>' % (os.path.basename(__file__), smtp_creds.user)): 101) """ Sends Email """ 102) smtp = smtplib.SMTP(smtp_creds.server, port=smtp_creds.port) 103) smtp.login(smtp_creds.user, smtp_creds.passw)`
Better conformance to PEP-8... David Blume authored 6 years ago	`104) smtp.sendmail(fromaddr, 105) toaddrs,`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`106) "Content-Type: text/plain; charset=\"us-ascii\"\r\nFrom: %s\r\nTo: %s\r\nSubject: %s\r\n%s" % \ 107) (fromaddr, ", ".join(toaddrs), subject, message))`
Original 2010-09-03 version David Blume authored 6 years ago	`108) smtp.quit() 109)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`110)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`111) def index_id(a_list, op, elem):`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`112) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`113) return (index for index, item in enumerate(a_list) if op(item, elem)).next()`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`114) except: 115) return -1 116)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`117)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`118) def make_chart_url(time_posted, lhs_times, lhs_values, rhs_times, 119) rhs_values, threshold_value, is_odd_row, tag_hit): 120) # lhs_times, lhs_values = zip(comments) 121) # rhs_times, rhs_values = zip(rhs)`
Original 2010-09-03 version David Blume authored 6 years ago	`122)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	123) if not len(lhs_times): 124) lhs_times = [time_posted,] 125) if not len(lhs_values): 126) lhs_values = [0,] 127) if not len(rhs_times): 128) rhs_times = [time_posted,] 129) if not len(rhs_values): 130) rhs_values = [0,] 131) 132) lhs_times = [(i - time_posted) / 1800 for i in lhs_times] 133) rhs_times = [(i - time_posted) / 1800 for i in rhs_times] 134) 135) max_comment_time = max(lhs_times) 136) max_comment_value = max(lhs_values) 137) max_rhs_time = max(rhs_times) 138) max_rhs_value = max(rhs_values)
Original 2010-09-03 version David Blume authored 6 years ago	`139)`
2011-02-04: Update to the c... David Blume authored 6 years ago	`140) met_threshold_pt = -1 141) if threshold_value != -1:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`142) met_threshold_pt = index_id(rhs_values, operator.ge, threshold_value)`
2011-02-04: Update to the c... David Blume authored 6 years ago	`143) if met_threshold_pt == -1 or tag_hit: 144) # This can happen if threshold_value was set to a number 145) # because the author or a tag was matched, but the article 146) # was unpopular. We choose to put a marker at point index 0. 147) met_threshold_pt = 0 148) 149) if is_odd_row != 0: 150) bg_color = even_background 151) watermark_color = even_watermark 152) else: 153) bg_color = odd_background 154) watermark_color = odd_watermark 155)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`156) if len(lhs_values) < 8 and len(lhs_values) > 1:`
Original 2010-09-03 version David Blume authored 6 years ago	`157) # max_comment_value *= 2 158) pass`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`159) elif len(lhs_values) == 1:`
Original 2010-09-03 version David Blume authored 6 years ago	`160) min_comment_value = 0`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`161) if len(rhs_values) < 8 and len(rhs_values) > 1: 162) # max_rhs_value *= 2`
Original 2010-09-03 version David Blume authored 6 years ago	`163) pass`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`164) elif len(rhs_values) == 1: 165) min_rhs_value = 0`
Original 2010-09-03 version David Blume authored 6 years ago	`166) 167) min_comment_value = 0`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`168) min_rhs_value = 0`
Original 2010-09-03 version David Blume authored 6 years ago	`169) 170) chart_url = "http://chart.apis.google.com/chart?cht=lxy&chco=%s,%s&chs=%dx%d&chxs=0,%s\|1,%s" % \`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`171) (series_1_color, series_2_color, img_width, img_height, series_1_color, series_2_color) 172) chart_url += "&chd=t:%s\|%s\|%s\|%s" % (','.join([str(n) for n in lhs_times]), 173) ','.join([str(n) for n in lhs_values]), 174) ','.join([str(n) for n in rhs_times]), 175) ','.join([str(n) for n in rhs_values]))`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`176) # TODO: Consider watermark levels, like: 177) # chm=h,B0B0B0,1,0.3,1\|r,E0E0E0,0,0,0.5`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`178) if max_rhs_value > 0: 179) threshold_percent = max(0, min((float(threshold_value) / max_rhs_value) - 0.01, 1.0))`
2011-02-04: Update to the c... David Blume authored 6 years ago	`180) else: 181) threshold_percent = 1.0`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`182) chart_url += "&chm=r,%s,0,0,%1.3f" % (watermark_color, threshold_percent)`
Original 2010-09-03 version David Blume authored 6 years ago	`183) if met_threshold_pt != -1:`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`184) if tag_hit: 185) dot_color = tag_color 186) dot_shape = 'd' 187) else: 188) dot_color = threshold_color 189) dot_shape = 'o'`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`190) chart_url += "\|%s,%s,1,%d,10" % (dot_shape, dot_color, met_threshold_pt)`
Original 2010-09-03 version David Blume authored 6 years ago	`191) chart_url += "&chxt=y,r&chxl=0:\|%d\|%d\|1:\|%d\|%d&chds=%d,%d,%d,%d,%d,%d,%d,%d" % \`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`192) (min_comment_value, max_comment_value, min_rhs_value, max_rhs_value, 193) 0, max(7, max_comment_time), 194) min_comment_value, max_comment_value, 195) 0, max(7, max_rhs_time), 196) min_comment_value, max_rhs_value) 197) chart_url += "&chf=bg,s,%s&chdl=comments\|shares" % (bg_color,)`
Original 2010-09-03 version David Blume authored 6 years ago	`198) return chart_url 199)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`200)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`201) def process_feed(yaml_items):`
Moved SMTP credentials to s... David Blume authored 6 years ago	`202) """Retrieve the url and process it.`
Original 2010-09-03 version David Blume authored 6 years ago	`203) feed_info (in, out) A tuple that describes an individual feed, like its name and etag. 204) """`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`205) feed = feedparser.parse('http://feeds.feedburner.com/TechCrunch') 206) if hasattr(feed, 'status'):`
Original 2010-09-03 version David Blume authored 6 years ago	207) if feed.status == 304: 208) pass 209) else: 210) if feed.status != 200 and feed.status != 307 and feed.status != 301 and feed.status != 302: 211) if feed.status == 503: 212) print "the feed is temporarily unavailable." 213) elif feed.status == 400: 214) print "the feed says we made a bad request." 215) elif feed.status == 502: 216) print "the feed reported a bad gateway error." 217) elif feed.status == 404: 218) print "the feed says the page was not found." 219) elif feed.status == 500: 220) print "the feed had an internal server error." 221) elif feed.status == 403: 222) print "Access to the feed was forbidden." 223) else: 224) print "the feed returned feed.status %d." % ( feed.status, ) 225) else: 226) # Save off this
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`227) if hasattr(feed, 'bozo_exception') and isinstance(feed.bozo_exception, xml.sax._exceptions.SAXParseException): 228) print "Didn't pickle TechCrunch feed because it had a bozo_exception: %s" % (str(feed.bozo_exception))`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`229) else: 230) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`231) with open(os.path.join(localdir, 'techcrunch_feed.pickle'), 'wb') as f: 232) pickle.dump(feed, f)`
Catch up to production agai... David Blume authored 6 years ago	`233) except(pickle.PicklingError, exceptions.TypeError) as e:`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`234) print "An error occurred while pickling the feed: %s." % \`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`235) (# str(e.__class__), 236) str(e)) 237) traceback.print_exc(3, file=sys.stdout)`
Original 2010-09-03 version David Blume authored 6 years ago	`238)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`239) for i in reversed(feed.entries): 240) process_item(i, yaml_items)`
Original 2010-09-03 version David Blume authored 6 years ago	`241) 242) # If we have more than 200 items, remove the old ones.`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`243) while len(yaml_items) > 200:`
Original 2010-09-03 version David Blume authored 6 years ago	`244) yaml_items.pop() 245) 246) for i in yaml_items:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`247) # i['title'] = asciiize(i['title']) 248) # i['tags'] = map(asciiize, i['tags']) 249) process_yaml_item(i)`
Original 2010-09-03 version David Blume authored 6 years ago	`250) 251) else: 252) if hasattr(feed, 'bozo_exception'): 253) e = feed.bozo_exception`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`254) if isinstance(e, urllib2.URLError):`
Original 2010-09-03 version David Blume authored 6 years ago	`255) print_last_line = True 256) if hasattr(e, 'reason'): 257) if e.reason[0] == 110: 258) print "the feed's connection timed out." 259) print_last_line = False 260) elif e.reason[0] == 111: 261) print "the feed's connection was refused." 262) print_last_line = False 263) elif e.reason[0] == 104: 264) print "the feed reset the connection." 265) print_last_line = False 266) else:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`267) print "the feed had a URLError with reason %s." % (str(e.reason),)`
Original 2010-09-03 version David Blume authored 6 years ago	`268) print_last_line = False 269) if print_last_line:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`270) print "the feed had a URLError %s" % (str(e),) 271) elif isinstance(e, httplib.BadStatusLine): 272) print "the feed gave a bad status line. (%s)" % (str(e),)`
Original 2010-09-03 version David Blume authored 6 years ago	`273) else:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`274) if len(str(e)): 275) print "the feed bozo_exception: %s \"%s\"" % (str(e.__class__), str(e))`
Original 2010-09-03 version David Blume authored 6 years ago	`276) else:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`277) print "the feed bozo_exception: %s %s" % (str(e.__class__), repr(e))`
Original 2010-09-03 version David Blume authored 6 years ago	`278) else:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`279) print "the feed returned class %s, %s" % (str(feed.__class__), str(feed))`
Original 2010-09-03 version David Blume authored 6 years ago	`280)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`281)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`282) def process_item(feed_item, yaml_items):`
Catch up to production agai... David Blume authored 6 years ago	`283) """Processes an RSS feed item, and converts it to a YAML item"""`
Original 2010-09-03 version David Blume authored 6 years ago	`284) # Get the time 285) global any_entry_added`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`286) timecode_now = int(time.time())`
Original 2010-09-03 version David Blume authored 6 years ago	`287) date_parsed = time.gmtime()`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`288) if hasattr(feed_item, 'issued_parsed'):`
Original 2010-09-03 version David Blume authored 6 years ago	`289) date_parsed = feed_item.issued_parsed`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`290) elif hasattr(feed_item, 'date_parsed'):`
Original 2010-09-03 version David Blume authored 6 years ago	`291) date_parsed = feed_item.date_parsed 292) else:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`293) print "process_item found no timestamp for", asciiize(feed_item.link) 294) timecode_parsed = calendar.timegm(date_parsed)`
Original 2010-09-03 version David Blume authored 6 years ago	`295)`
Catch up to production agai... David Blume authored 6 years ago	`296) link = feed_item.link 297) if hasattr(feed_item, 'feedburner_origlink'): 298) link = feed_item.feedburner_origlink 299) 300) # TODO 2018-01-18: Leave in the ncid for URL clicks, but remove during processing. 301) # suffix_to_remove = '?ncid=rss' 302) # if link.endswith(suffix_to_remove): 303) # link = link[:-len(suffix_to_remove)] 304)`
Original 2010-09-03 version David Blume authored 6 years ago	`305) # Look for i.feedburner_origlink in yaml_items 306) yaml_item = None 307) for i in yaml_items:`
Catch up to production agai... David Blume authored 6 years ago	`308) if link == i['link']:`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`309) yaml_item = i 310) break`
Catch up to production agai... David Blume authored 6 years ago	`311) if yaml_item is None:`
Original 2010-09-03 version David Blume authored 6 years ago	`312) author = ''`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`313) if hasattr(feed_item, 'author'): 314) author = asciiize(feed_item.author)`
Original 2010-09-03 version David Blume authored 6 years ago	`315)`
Catch up to production agai... David Blume authored 6 years ago	`316) # Make a new yaml_item`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`317) yaml_item = {'title' : asciiize(feed_item.title), 318) 'link' : asciiize(link), 319) 'author' : author, 320) 'tags' : [], 321) 'orig_posted' : timecode_parsed, 322) 'qualified' : -1, 323) 'comment_times' : [], 324) 'fb_comments' : [], 325) 'fb_shares' : [], 326) 'fb_likes' : [], 327) 'slash_comment_times' : [], 328) 'slash_comments' : []`
Original 2010-09-03 version David Blume authored 6 years ago	`329) }`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`330) if hasattr(feed_item, 'tags'):`
Original 2010-09-03 version David Blume authored 6 years ago	`331) for i in feed_item.tags:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`332) yaml_item['tags'].append(asciiize(i.term))`
Original 2010-09-03 version David Blume authored 6 years ago	`333)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`334) yaml_items.insert(0, yaml_item)`
Original 2010-09-03 version David Blume authored 6 years ago	`335) any_entry_added = True 336) 337) # Maybe check to ensure that this item isn't too old. 338) if timecode_parsed < timecode_now - 60 * 30 * 9: 339) return 340) 341) # Now, add the new values`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`342) if hasattr(feed_item, 'slash_comments') and len(yaml_item['slash_comments']) < 8:`
Original 2010-09-03 version David Blume authored 6 years ago	`343) any_entry_added = True`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`344) yaml_item['slash_comment_times'].append(timecode_now) 345) yaml_item['slash_comments'].append(int(feed_item.slash_comments))`
Original 2010-09-03 version David Blume authored 6 years ago	`346)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`347)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`348) def process_yaml_item(yaml_item):`
Original 2010-09-03 version David Blume authored 6 years ago	`349) global any_entry_added 350)`
Catch up to production agai... David Blume authored 6 years ago	`351) # Related to TODO 2018-01-18: Remove ncid only during processing. 352) link = yaml_item['link'] 353) suffix_to_remove = '?ncid=rss' 354) # Maybe we should find() it instead, in case feedburner adds other options 355) if link.endswith(suffix_to_remove): 356) link = link[:-len(suffix_to_remove)] 357)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`358) timecode_now = int(time.time()) 359) if len(yaml_item['fb_comments']) < 8:`
Catch up to production agai... David Blume authored 6 years ago	`360) num_shares, num_comments, num_likes = Get_fb_stats(link)`
Original 2010-09-03 version David Blume authored 6 years ago	`361) if num_comments != -1: 362) any_entry_added = True`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`363) yaml_item['comment_times'].append(timecode_now) 364) yaml_item['fb_shares'].append(num_shares) 365) yaml_item['fb_comments'].append(num_comments) 366) yaml_item['fb_likes'].append(num_likes)`
Original 2010-09-03 version David Blume authored 6 years ago	`367)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`368) # if len(yaml_item['reddit_']) < 8:`
Better conformance to PEP-8... David Blume authored 6 years ago	`369) # num_ = get_reddit_stats(link)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`370) # if num_ != -1: 371) # any_entry_added = True 372) # yaml_item['reddit_times'].append(timecode_now) 373) # yaml_item['reddit_'].append(num_)`
Original 2010-09-03 version David Blume authored 6 years ago	`374)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`375)`
Better conformance to PEP-8... David Blume authored 6 years ago	`376) def get_reddit_stats(url_string):`
Catch up to production agai... David Blume authored 6 years ago	`377) """ Consider curl "https://www.reddit.com/api/info.json?url=http://i.imgur.com/HG9dJ.jpg"`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`378) """`
Original 2010-09-03 version David Blume authored 6 years ago	`379) return -1 380)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`381)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`382) def Get_fb_stats(url_string):`
Moved SMTP credentials to s... David Blume authored 6 years ago	`383) """Use graph's "engagement" field to get reactions and shares."""`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`384) shares = -1 385) comments = -1`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`386) likes = -1 387)`
Catch up to production agai... David Blume authored 6 years ago	`388) url_string = url_string.encode('utf-8') 389)`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`390) try:`
Catch up to production agai... David Blume authored 6 years ago	`391) encoded = urllib.urlencode({'access_token': facebook_token}) 392) url = 'https://graph.facebook.com/v2.11/?id=%s&fields=engagement&%s' 393) f = urllib2.urlopen(url % (urllib.quote_plus(url_string), encoded))`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`394) data = f.read() 395) f.close()`
Catch up to production agai... David Blume authored 6 years ago	`396) except (urllib2.URLError, httplib.BadStatusLine) as e:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`397) if hasattr(e, 'reason'): # URLError`
Catch up to production agai... David Blume authored 6 years ago	`398) if hasattr(e, 'code'): 399) print "Get_fb_stats got an error (1):", e.code, e.reason, url_string 400) else: 401) print "Get_fb_stats got an error (2):", e.reason, url_string`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`402) elif hasattr(e, 'code'): #URLError`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`403) print "Get_fb_stats got an error. Code:", e.code, url_string 404) else:`
Catch up to production agai... David Blume authored 6 years ago	`405) print "Get_fb_stats got an error (3):", str(e) 406) return shares, comments, likes`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`407) if len(data) > 20:`
Catch up to production agai... David Blume authored 6 years ago	`408) d = json.loads(data)['engagement'] 409) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`410) shares = d['share_count']`
Catch up to production agai... David Blume authored 6 years ago	`411) except KeyError:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`412) shares = 0`
Catch up to production agai... David Blume authored 6 years ago	`413) 414) try: 415) likes = d['reaction_count'] 416) except KeyError: 417) likes = 0 418) 419) # TODO 2018-01-18: og_object metric was likes + shares + comments 420) # Here we'll combine likes and shares, and comments with plugin_comments 421) likes += shares 422) 423) try: 424) comments = d['comment_plugin_count'] + d['comment_count'] 425) except KeyError: 426) comments = 0`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`427) else: 428) print "Get_fb_stats got too little data for ", url_string`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`429) return shares, comments, likes`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`430) 431)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`432) def save_image(url_string, file_path):`
Original 2010-09-03 version David Blume authored 6 years ago	`433) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`434) f = urllib2.urlopen(url_string)`
Original 2010-09-03 version David Blume authored 6 years ago	`435) data = f.read() 436) f.close()`
Catch up to production agai... David Blume authored 6 years ago	`437) except (urllib2.URLError, httplib.BadStatusLine) as e:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`438) if hasattr(e, 'reason'): # URLError`
Moved SMTP credentials to s... David Blume authored 6 years ago	`439) print "save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Reason:", e.reason`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`440) elif hasattr(e, 'code'): # URLError`
Moved SMTP credentials to s... David Blume authored 6 years ago	`441) print "save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Code:", e.code`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`442) else:`
Moved SMTP credentials to s... David Blume authored 6 years ago	`443) print "save_image: Error from urlopen", e`
Original 2010-09-03 version David Blume authored 6 years ago	`444) return url_string`
2013-08-04: Miscellaneous c... David Blume authored 6 years ago	`445)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`446) if len(data) > 50: 447) with open(file_path, 'wb') as f: 448) f.write(data) 449) return 'cache/' + os.path.basename(file_path)`
Original 2010-09-03 version David Blume authored 6 years ago	`450) return url_string 451)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`452)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`453) def make_index_html(yaml_items, weekend_stats, weekday_stats): 454) """Writes a static index.html file from the YAML items."""`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`455) cur_time = int(time.time()) 456) new_index_fullpath = os.path.join(localdir, 'index.html_new') 457) index_fullpath = os.path.join(localdir, 'index.html') 458) cache_path = os.path.join(localdir, 'cache')`
Original 2010-09-03 version David Blume authored 6 years ago	`459)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`460) files_to_delete = glob.glob(os.path.join(cache_path, '*.png'))`
Original 2010-09-03 version David Blume authored 6 years ago	`461)`
Catch up to production agai... David Blume authored 6 years ago	462) with codecs.open(new_index_fullpath, 'w', 'utf-8') as f: 463) f.write(html_head % (even_background, odd_background)) 464) 465) f.write('<div align="center">\n<table class="legend">\n<tr><th></th><th>Median</th><th>Mean</th><th>Std. Dev</th><th>Threshold</th></tr>\n') 466) f.write('<tr><th>Weekday</th><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td></tr>\n' % (weekday_stats[2][0], weekday_stats[2][1], weekday_stats[2][2], weekday_stats[2][1] + weekday_stats[2][2])) 467) f.write('<tr><th>Weekend</th><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td></tr>\n' % (weekend_stats[2][0], weekend_stats[2][1], weekend_stats[2][2], weekend_stats[2][1] + weekend_stats[2][2])) 468) f.write('</table></div>\n<br />\n') 469) 470) f.write('<div align="center">\n<table>\n') 471) for image_index, image in enumerate(yaml_items[:40]): 472) tag_hit = False 473) if image['author'].lower() in authors_to_post: 474) tag_hit = True 475) elif len(set([j.lower() for j in image['tags']]) & tags_to_post) > 0: 476) tag_hit = True 477) chart_url = make_chart_url(image['orig_posted'], 478) image['comment_times'], 479) image['fb_comments'], 480) image[rhs_metric_times], 481) image[rhs_metric], 482) image['qualified'], 483) image_index % 2, 484) tag_hit 485) )
Moved SMTP credentials to s... David Blume authored 6 years ago	`486) image_url = save_image(chart_url, os.path.join(cache_path, '%d_%d.png' % (cur_time, image_index)))`
Catch up to production agai... David Blume authored 6 years ago	487) f.write('<tr valign="center" class="%s">\n <td><strong><a href="%s">%s</a></strong> <span class="author">by %s</span></td>\n' % \ 488) (image_index % 2 and "even" or "odd", 489) image['link'], 490) image['title'].encode('ascii', 'xmlcharrefreplace'), 491) image['author'].encode('ascii', 'xmlcharrefreplace'), 492) ) 493) ) 494) f.write(' <td>%s<td>\n' % (image['qualified'] != -1 and '<img src="star_30.png" width="30" height="29" />' or '')) 495) f.write(' <td><img src="%s" width="%d" height="%d" border="0" /></td></tr>\n' % \ 496) (image_url, 497) img_width, 498) img_height 499) ) 500) ) 501) f.write(html_footer) 502)
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`503) if os.path.exists(index_fullpath): 504) os.unlink(index_fullpath) 505) shutil.move(new_index_fullpath, index_fullpath)`
Original 2010-09-03 version David Blume authored 6 years ago	`506) for fname in files_to_delete:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`507) os.unlink(fname)`
Original 2010-09-03 version David Blume authored 6 years ago	`508)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`509)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`510) def make_feed_file(yaml_items): 511) """Writes the RSS feed file with the YAML items."""`
Catch up to production agai... David Blume authored 6 years ago	`512) with codecs.open(os.path.join(localdir, 'rss_feed.xml'), 'wb', 'utf-8') as f: 513) f.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<rss version=\"2.0\">\n<channel>\n<title>Trending at TechCrunch</title><link>http://techcrunch.dlma.com</link>")`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`514) f.write("<pubDate>%s</pubDate><description>Automatically Generated Feed</description><language>en-us</language>\n" % (time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())))`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`515) count = 0 516) for item in yaml_items:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`517) now = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(item['orig_posted']))`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`518) if item['qualified'] != -1:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`519) escaped_title = cgi.escape(item['title']).encode('ascii', 'xmlcharrefreplace') 520) escaped_author = cgi.escape(item['author']).encode('ascii', 'xmlcharrefreplace') 521) f.write("<item><title>%s</title><pubDate>%s</pubDate><link>%s</link><guid isPermaLink=\"false\">%s</guid><description><![CDATA[By: %s]]></description></item>\n" % \ 522) (escaped_title, now, item['link'], item['link'], escaped_author))`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`523) count += 1 524) if count > 14: 525) break`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`526) f.write("</channel></rss>")`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`527)`
Original 2010-09-03 version David Blume authored 6 years ago	`528) 529) if __name__=='__main__': 530) start_time = time.time() 531) progress_text = [] 532) 533) old_stdout = sys.stdout 534) old_stderr = sys.stderr 535) sys.stdout = sys.stderr = StringIO.StringIO() 536) 537) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`538) localdir = os.path.abspath(os.path.dirname(sys.argv[0]))`
Original 2010-09-03 version David Blume authored 6 years ago	`539) # 540) # Read in techcrunch.yaml 541) # 542) # [ { 'title' : 'Title Text', 543) # 'link' : u'http://techcrunch.com/2010/08/17/google-buzz-who-to-follow/', 544) # 'author' : u'MG Siegler', 545) # 'orig_posted' : 1282197199 546) # 'tags' : [ u'Google', u'privacy' ] 547) # 'qualified' : -1 548) # 'comment_times' : [ 1282197199, 1282197407 ]`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`549) # 'fb_comments' : [ 0, 5 ] 550) # 'fb_shares' : [ 0, 300 ] 551) # 'fb_likes' : [ 0, 19 ]`
Original 2010-09-03 version David Blume authored 6 years ago	`552) # 'slash_comment_times' : [ 1282197199, 1282197407 ] 553) # 'slash_comments' : [ 0, 5 ] 554) # }, 555) # { ... } 556) # ] 557) #`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`558) yaml_fullpath = os.path.join(localdir, 'techcrunch.yaml') 559) if os.path.exists(yaml_fullpath): 560) with open(yaml_fullpath, 'rb') as f: 561) items = yaml.load(f)`
Catch up to production agai... David Blume authored 6 years ago	`562) if items is None: 563) print yaml_fullpath, "exists, but was empty." 564) items = []`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`565) 566) # Do any dictionary item updating that might be necessary 567) # for item in items:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`568) # if not item.has_key('fb_shares'):`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`569) # item['fb_shares'] = []`
Original 2010-09-03 version David Blume authored 6 years ago	`570) else: 571) print "could not open", yaml_fullpath 572) items = [] 573)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`574) with open(os.path.join(localdir, 'facebook-token.txt'), 'r') as f:`
Catch up to production agai... David Blume authored 6 years ago	`575) facebook_token = f.read() 576)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`577) progress_text = ["read techcrunch.yaml"] 578) process_feed(items)`
Original 2010-09-03 version David Blume authored 6 years ago	`579) 580) # 581) # If any work was done, then write files. 582) #`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`583) if any_entry_added:`
Better conformance to PEP-8... David Blume authored 6 years ago	`584) weekend_stats, weekday_stats = analysis.process_feed(items, rhs_metric, rhs_metric_times)`
Original 2010-09-03 version David Blume authored 6 years ago	`585) 586) # We'll only look at the stats for the time 1:00 to 1:30 after posting.`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`587) weekend_median, weekend_mean, weekend_sigma = weekend_stats[2] 588) weekend_threshold = weekend_mean + weekend_sigma 589) weekday_median, weekday_mean, weekday_sigma = weekday_stats[2] 590) weekday_threshold = weekday_mean + weekday_sigma 591) for item in items:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`592) wday = time.localtime(item['orig_posted']).tm_wday`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`593) if wday == 5 or wday == 6: 594) threshold = weekend_threshold 595) else: 596) threshold = weekday_threshold 597) if item['qualified'] == -1:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`598) for i in range(len(item[rhs_metric_times])): 599) r_time = item[rhs_metric_times][i]`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`600) if r_time - item['orig_posted'] < 5400:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`601) if item[rhs_metric][i] >= threshold:`
2011-02-04: Update to the c... David Blume authored 6 years ago	`602) item['qualified'] = threshold`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`603) if r_time - item['orig_posted'] >= 3600: 604) break 605) 606) # Automatically add those items whose authors and tags I like 607) for item in items:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`608) if item['qualified'] == -1 and len(item[rhs_metric_times]) > 0:`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`609) if item['author'].lower() in authors_to_post:`
2011-02-04: Update to the c... David Blume authored 6 years ago	`610) item['qualified'] = threshold`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`611) elif len(set([j.lower() for j in item['tags']]) & tags_to_post) > 0:`
2011-02-04: Update to the c... David Blume authored 6 years ago	`612) item['qualified'] = threshold`
Original 2010-09-03 version David Blume authored 6 years ago	`613) 614) # 615) # Write out the updated yaml file. 616) #`
2011-02-04: Algorithm chang... David Blume authored 6 years ago	`617) 618) # For the one file we really use, write to a file on the side, then move it.`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`619) yaml_newfile_fullpath = os.path.join(localdir, 'techcrunch_temp_writable.yaml') 620) with open(yaml_newfile_fullpath, 'wb') as f: 621) yaml.dump(items, f, width=120)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`622) try:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`623) os.rename(yaml_newfile_fullpath, yaml_fullpath)`
2015-11-23: Resync svn with... David Blume authored 6 years ago	`624) except OSError as e: 625) print "The source file was", yaml_newfile_fullpath, "and exists =", os.path.isfile(yaml_newfile_fullpath)`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`626) with open(os.path.join(localdir, 'techcrunch_text.yaml'), 'w') as f: 627) yaml.dump(items, f, width=120) 628) with codecs.open(os.path.join(localdir, 'techcrunch_unicode.yaml'), 'w', 'utf-8') as f: 629) yaml.dump(items, f, encoding='utf-8', width=120)`
Original 2010-09-03 version David Blume authored 6 years ago	`630)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`631) make_feed_file(items)`
Original 2010-09-03 version David Blume authored 6 years ago	`632)`
Moved SMTP credentials to s... David Blume authored 6 years ago	`633) make_index_html(items, weekend_stats, weekday_stats)`
Original 2010-09-03 version David Blume authored 6 years ago	`634) else: 635) print "No entries were added this time." 636)`
Catch up to production agai... David Blume authored 6 years ago	`637) except Exception as e:`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`638) exceptional_text = "An exception occurred: " + str(e.__class__) + " " + str(e) 639) print exceptional_text, ' '.join(progress_text) 640) traceback.print_exc(file=sys.stdout)`
Original 2010-09-03 version David Blume authored 6 years ago	`641) try:`
Moved SMTP credentials to s... David Blume authored 6 years ago	`642) send_email('Exception thrown in techcrunch.py',`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`643) exceptional_text + "\n" + traceback.format_exc(), 644) ('david.blume@gmail.com',))`
Catch up to production agai... David Blume authored 6 years ago	`645) except Exception as e:`
Original 2010-09-03 version David Blume authored 6 years ago	`646) print "Could not send email to notify you of the exception. :(" 647) 648) message = sys.stdout.getvalue() 649) sys.stdout = old_stdout 650) sys.stderr = old_stderr 651) if not debug: 652) print message 653) 654) # Finally, let's save this to a statistics page`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`655) if os.path.exists(os.path.join(localdir, 'stats.txt')): 656) with open(os.path.join(localdir, 'stats.txt')) as f:`
Original 2010-09-03 version David Blume authored 6 years ago	`657) lines = f.readlines() 658) else: 659) lines = []`
Catch up to production agai... David Blume authored 6 years ago	`660) lines = lines[:672] # Just keep the past week's worth`
2015-11-27: Remove obsolete... David Blume authored 6 years ago	`661) # status = len(message.strip()) and message.strip().replace('\n', ' - ') or "OK" 662) status = len(message.strip()) and '\n '.join( message.splitlines()) or "OK" 663) lines.insert(0, "%s %3.0fs %s\n" % (time.strftime('%Y-%m-%d, %H:%M', time.localtime()), time.time() - start_time, status)) 664) with open(os.path.join(localdir,'stats.txt' ), 'w') as f: 665) f.writelines(lines)`