"""
img_width = 300
img_height = 50
series_1_color = "0000FF"
series_2_color = "00AA00"
threshold_color = "FF8C00"
tag_color = "F01000"
even_background = "F8F8F8"
odd_background = "E8E8E8"
even_watermark = "E0E0FF"
odd_watermark = "D0D0F0"
def asciiize(s):
try:
return s.encode('ascii')
except UnicodeEncodeError:
return s
except exceptions.AttributeError:
return s
def send_email(subject, message, toaddrs,
fromaddr='"%s" <%s>' % (os.path.basename(__file__), smtp_creds.user)):
""" Sends Email """
smtp = smtplib.SMTP(smtp_creds.server, port=smtp_creds.port)
smtp.login(smtp_creds.user, smtp_creds.passw)
smtp.sendmail(fromaddr,
toaddrs,
"Content-Type: text/plain; charset=\"us-ascii\"\r\nFrom: %s\r\nTo: %s\r\nSubject: %s\r\n%s" % \
(fromaddr, ", ".join(toaddrs), subject, message))
smtp.quit()
def index_id(a_list, op, elem):
try:
return (index for index, item in enumerate(a_list) if op(item, elem)).next()
except:
return -1
def make_chart_url(time_posted, lhs_times, lhs_values, rhs_times,
rhs_values, threshold_value, is_odd_row, tag_hit):
# lhs_times, lhs_values = zip(*comments)
# rhs_times, rhs_values = zip(*rhs)
if not len(lhs_times):
lhs_times = [time_posted,]
if not len(lhs_values):
lhs_values = [0,]
if not len(rhs_times):
rhs_times = [time_posted,]
if not len(rhs_values):
rhs_values = [0,]
lhs_times = [(i - time_posted) / 1800 for i in lhs_times]
rhs_times = [(i - time_posted) / 1800 for i in rhs_times]
max_comment_time = max(lhs_times)
max_comment_value = max(lhs_values)
max_rhs_time = max(rhs_times)
max_rhs_value = max(rhs_values)
met_threshold_pt = -1
if threshold_value != -1:
met_threshold_pt = index_id(rhs_values, operator.ge, threshold_value)
if met_threshold_pt == -1 or tag_hit:
# This can happen if threshold_value was set to a number
# because the author or a tag was matched, but the article
# was unpopular. We choose to put a marker at point index 0.
met_threshold_pt = 0
if is_odd_row != 0:
bg_color = even_background
watermark_color = even_watermark
else:
bg_color = odd_background
watermark_color = odd_watermark
if len(lhs_values) < 8 and len(lhs_values) > 1:
# max_comment_value *= 2
pass
elif len(lhs_values) == 1:
min_comment_value = 0
if len(rhs_values) < 8 and len(rhs_values) > 1:
# max_rhs_value *= 2
pass
elif len(rhs_values) == 1:
min_rhs_value = 0
min_comment_value = 0
min_rhs_value = 0
chart_url = "http://chart.apis.google.com/chart?cht=lxy&chco=%s,%s&chs=%dx%d&chxs=0,%s|1,%s" % \
(series_1_color, series_2_color, img_width, img_height, series_1_color, series_2_color)
chart_url += "&chd=t:%s|%s|%s|%s" % (','.join([str(n) for n in lhs_times]),
','.join([str(n) for n in lhs_values]),
','.join([str(n) for n in rhs_times]),
','.join([str(n) for n in rhs_values]))
# TODO: Consider watermark levels, like:
# chm=h,B0B0B0,1,0.3,1|r,E0E0E0,0,0,0.5
if max_rhs_value > 0:
threshold_percent = max(0, min((float(threshold_value) / max_rhs_value) - 0.01, 1.0))
else:
threshold_percent = 1.0
chart_url += "&chm=r,%s,0,0,%1.3f" % (watermark_color, threshold_percent)
if met_threshold_pt != -1:
if tag_hit:
dot_color = tag_color
dot_shape = 'd'
else:
dot_color = threshold_color
dot_shape = 'o'
chart_url += "|%s,%s,1,%d,10" % (dot_shape, dot_color, met_threshold_pt)
chart_url += "&chxt=y,r&chxl=0:|%d|%d|1:|%d|%d&chds=%d,%d,%d,%d,%d,%d,%d,%d" % \
(min_comment_value, max_comment_value, min_rhs_value, max_rhs_value,
0, max(7, max_comment_time),
min_comment_value, max_comment_value,
0, max(7, max_rhs_time),
min_comment_value, max_rhs_value)
chart_url += "&chf=bg,s,%s&chdl=comments|shares" % (bg_color,)
return chart_url
def process_feed(yaml_items):
"""Retrieve the url and process it.
feed_info (in, out) A tuple that describes an individual feed, like its name and etag.
"""
feed = feedparser.parse('http://feeds.feedburner.com/TechCrunch')
if hasattr(feed, 'status'):
if feed.status == 304:
pass
else:
if feed.status != 200 and feed.status != 307 and feed.status != 301 and feed.status != 302:
if feed.status == 503:
print "the feed is temporarily unavailable."
elif feed.status == 400:
print "the feed says we made a bad request."
elif feed.status == 502:
print "the feed reported a bad gateway error."
elif feed.status == 404:
print "the feed says the page was not found."
elif feed.status == 500:
print "the feed had an internal server error."
elif feed.status == 403:
print "Access to the feed was forbidden."
else:
print "the feed returned feed.status %d." % ( feed.status, )
else:
# Save off this
if hasattr(feed, 'bozo_exception') and isinstance(feed.bozo_exception, xml.sax._exceptions.SAXParseException):
print "Didn't pickle TechCrunch feed because it had a bozo_exception: %s" % (str(feed.bozo_exception))
else:
try:
with open(os.path.join(localdir, 'techcrunch_feed.pickle'), 'wb') as f:
pickle.dump(feed, f)
except(pickle.PicklingError, exceptions.TypeError) as e:
print "An error occurred while pickling the feed: %s." % \
(# str(e.__class__),
str(e))
traceback.print_exc(3, file=sys.stdout)
for i in reversed(feed.entries):
process_item(i, yaml_items)
# If we have more than 200 items, remove the old ones.
while len(yaml_items) > 200:
yaml_items.pop()
for i in yaml_items:
# i['title'] = asciiize(i['title'])
# i['tags'] = map(asciiize, i['tags'])
process_yaml_item(i)
else:
if hasattr(feed, 'bozo_exception'):
e = feed.bozo_exception
if isinstance(e, urllib2.URLError):
print_last_line = True
if hasattr(e, 'reason'):
if e.reason[0] == 110:
print "the feed's connection timed out."
print_last_line = False
elif e.reason[0] == 111:
print "the feed's connection was refused."
print_last_line = False
elif e.reason[0] == 104:
print "the feed reset the connection."
print_last_line = False
else:
print "the feed had a URLError with reason %s." % (str(e.reason),)
print_last_line = False
if print_last_line:
print "the feed had a URLError %s" % (str(e),)
elif isinstance(e, httplib.BadStatusLine):
print "the feed gave a bad status line. (%s)" % (str(e),)
else:
if len(str(e)):
print "the feed bozo_exception: %s \"%s\"" % (str(e.__class__), str(e))
else:
print "the feed bozo_exception: %s %s" % (str(e.__class__), repr(e))
else:
print "the feed returned class %s, %s" % (str(feed.__class__), str(feed))
def process_item(feed_item, yaml_items):
"""Processes an RSS feed item, and converts it to a YAML item"""
# Get the time
global any_entry_added
timecode_now = int(time.time())
date_parsed = time.gmtime()
if hasattr(feed_item, 'issued_parsed'):
date_parsed = feed_item.issued_parsed
elif hasattr(feed_item, 'date_parsed'):
date_parsed = feed_item.date_parsed
else:
print "process_item found no timestamp for", asciiize(feed_item.link)
timecode_parsed = calendar.timegm(date_parsed)
link = feed_item.link
if hasattr(feed_item, 'feedburner_origlink'):
link = feed_item.feedburner_origlink
# TODO 2018-01-18: Leave in the ncid for URL clicks, but remove during processing.
# suffix_to_remove = '?ncid=rss'
# if link.endswith(suffix_to_remove):
# link = link[:-len(suffix_to_remove)]
# Look for i.feedburner_origlink in yaml_items
yaml_item = None
for i in yaml_items:
if link == i['link']:
yaml_item = i
break
if yaml_item is None:
author = ''
if hasattr(feed_item, 'author'):
author = asciiize(feed_item.author)
# Make a new yaml_item
yaml_item = {'title' : asciiize(feed_item.title),
'link' : asciiize(link),
'author' : author,
'tags' : [],
'orig_posted' : timecode_parsed,
'qualified' : -1,
'comment_times' : [],
'fb_comments' : [],
'fb_shares' : [],
'fb_likes' : [],
'slash_comment_times' : [],
'slash_comments' : []
}
if hasattr(feed_item, 'tags'):
for i in feed_item.tags:
yaml_item['tags'].append(asciiize(i.term))
yaml_items.insert(0, yaml_item)
any_entry_added = True
# Maybe check to ensure that this item isn't too old.
if timecode_parsed < timecode_now - 60 * 30 * 9:
return
# Now, add the new values
if hasattr(feed_item, 'slash_comments') and len(yaml_item['slash_comments']) < 8:
any_entry_added = True
yaml_item['slash_comment_times'].append(timecode_now)
yaml_item['slash_comments'].append(int(feed_item.slash_comments))
def process_yaml_item(yaml_item):
global any_entry_added
# Related to TODO 2018-01-18: Remove ncid only during processing.
link = yaml_item['link']
suffix_to_remove = '?ncid=rss'
# Maybe we should find() it instead, in case feedburner adds other options
if link.endswith(suffix_to_remove):
link = link[:-len(suffix_to_remove)]
timecode_now = int(time.time())
if len(yaml_item['fb_comments']) < 8:
num_shares, num_comments, num_likes = Get_fb_stats(link)
if num_comments != -1:
any_entry_added = True
yaml_item['comment_times'].append(timecode_now)
yaml_item['fb_shares'].append(num_shares)
yaml_item['fb_comments'].append(num_comments)
yaml_item['fb_likes'].append(num_likes)
# if len(yaml_item['reddit_']) < 8:
# num_ = get_reddit_stats(link)
# if num_ != -1:
# any_entry_added = True
# yaml_item['reddit_times'].append(timecode_now)
# yaml_item['reddit_'].append(num_)
def get_reddit_stats(url_string):
""" Consider curl "https://www.reddit.com/api/info.json?url=http://i.imgur.com/HG9dJ.jpg"
"""
return -1
def Get_fb_stats(url_string):
"""Use graph's "engagement" field to get reactions and shares."""
shares = -1
comments = -1
likes = -1
url_string = url_string.encode('utf-8')
try:
encoded = urllib.urlencode({'access_token': facebook_token})
url = 'https://graph.facebook.com/v2.11/?id=%s&fields=engagement&%s'
f = urllib2.urlopen(url % (urllib.quote_plus(url_string), encoded))
data = f.read()
f.close()
except (urllib2.URLError, httplib.BadStatusLine) as e:
if hasattr(e, 'reason'): # URLError
if hasattr(e, 'code'):
print "Get_fb_stats got an error (1):", e.code, e.reason, url_string
else:
print "Get_fb_stats got an error (2):", e.reason, url_string
elif hasattr(e, 'code'): #URLError
print "Get_fb_stats got an error. Code:", e.code, url_string
else:
print "Get_fb_stats got an error (3):", str(e)
return shares, comments, likes
if len(data) > 20:
d = json.loads(data)['engagement']
try:
shares = d['share_count']
except KeyError:
shares = 0
try:
likes = d['reaction_count']
except KeyError:
likes = 0
# TODO 2018-01-18: og_object metric was likes + shares + comments
# Here we'll combine likes and shares, and comments with plugin_comments
likes += shares
try:
comments = d['comment_plugin_count'] + d['comment_count']
except KeyError:
comments = 0
else:
print "Get_fb_stats got too little data for ", url_string
return shares, comments, likes
def save_image(url_string, file_path):
try:
f = urllib2.urlopen(url_string)
data = f.read()
f.close()
except (urllib2.URLError, httplib.BadStatusLine) as e:
if hasattr(e, 'reason'): # URLError
print "save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Reason:", e.reason
elif hasattr(e, 'code'): # URLError
print "save_image: Error attempting to create", file_path[file_path.rfind('/')+1:], "Code:", e.code
else:
print "save_image: Error from urlopen", e
return url_string
if len(data) > 50:
with open(file_path, 'wb') as f:
f.write(data)
return 'cache/' + os.path.basename(file_path)
return url_string
def make_index_html(yaml_items, weekend_stats, weekday_stats):
"""Writes a static index.html file from the YAML items."""
cur_time = int(time.time())
new_index_fullpath = os.path.join(localdir, 'index.html_new')
index_fullpath = os.path.join(localdir, 'index.html')
cache_path = os.path.join(localdir, 'cache')
files_to_delete = glob.glob(os.path.join(cache_path, '*.png'))
with codecs.open(new_index_fullpath, 'w', 'utf-8') as f:
f.write(html_head % (even_background, odd_background))
f.write('