techcrunch.git

@@ -1,19 +1,18 @@

 #!/usr/bin/python

-# chmod 755 me, and make sure I have UNIX style newlines.

 #

 # techcrunch.py

 #

-# http://feeds.feedburner.com/TechCrunch

-# feed = feedparser.parse( 'http://feeds.feedburner.com/TechCrunch' )

-# feed.entries[14]['feedburner_origlink'], feed.entries[14]['slash_comments']

+# For reference: See the SVN history of this file to see how I implemented

+# 1. retweet counts

+# 2. slash comment counts

+# 3. disqus comment counts, and the cookie for them

+# http://websvn.dlma.com/filedetails.php?repname=private&path=%2Fwww%2Ftechcrunch.dlma.com%2Ftrunk%2Ftechcrunch.py

 #

 # TODO:

 # 1. Deep links: '<a class="post_more_link snap_nopreview" href="http://www.crunchgear.com/2010/08/21/fanboyism-when-expression-meets-desperation/">Read the rest of this entry &raquo;</a>'

 #   link = "http://techcrunch.com/2010/08/21/fanboyism-when-expression-meets-desperation/"

 # 2. Add Reddit counts: curl "http://www.reddit.com/api/info.json?url=http://i.imgur.com/HG9dJ.jpg"

 #

-# This file was coverted from tabs to spaces with the vim command %retab

-#

 # cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; cp techcrunch.yaml techcrunch_tmp.yaml; cp techcrunch.yaml_back techcrunch.yaml

 #

@@ -34,10 +33,8 @@ import httplib

 import shutil

 import glob

 import smtplib

-import bisect

 import analysis

 import json

-import cookielib

 import xml

 import texttime

 import operator

@@ -49,6 +46,9 @@ any_entry_added = False

 tags_to_post = set(['apple', 'google'])

 authors_to_post = ['michael arrington',]

+rhs_metric = 'fb_shares'

+rhs_metric_times = 'comment_times'

+

 localdir = ''

 html_head = """

@@ -127,48 +127,39 @@ def index_id( a_list, op, elem ):

         return -1

-def index_id_simple( a_list, elem ):

-    index = 0

-    for item in a_list:

-        if item == elem:

-            return index

-        index += 1

-    return -1

-

-

-def make_chart_url( time_posted, comment_times, comment_values, retweet_times,

-                    retweet_values, threshold_value, is_odd_row, tag_hit ):

-#    comment_times, comment_values = zip( *comments )

-#    retweet_times, retweet_values = zip( *retweets )

+def make_chart_url(time_posted, lhs_times, lhs_values, rhs_times,

+                   rhs_values, threshold_value, is_odd_row, tag_hit):

+#    lhs_times, lhs_values = zip(*comments)

+#    rhs_times, rhs_values = zip(*rhs)

     # TODO handle failure cases, -1

-    if not len( comment_times ):

-        comment_times = [ time_posted, ]

-    if not len( comment_values ):

-        comment_values = [ 0, ]

-    if not len( retweet_times ):

-        retweet_times = [ time_posted, ]

-    if not len( retweet_values ):

-        retweet_values = [ 0, ]

-

-#    comment_times = [ (i - time_posted + 900) / 1800 for i in comment_times ]

-#    retweet_times = [ (i - time_posted + 900) / 1800 for i in retweet_times ]

-    comment_times = [ (i - time_posted) / 1800 for i in comment_times ]

-    retweet_times = [ (i - time_posted) / 1800 for i in retweet_times ]

-

-    min_comment_time = min( comment_times )

-    max_comment_time = max( comment_times )

-    min_comment_value = min( comment_values )

-    max_comment_value = max( comment_values )

-    min_retweet_time = min( retweet_times )

-    max_retweet_time = max( retweet_times )

-    min_retweet_value = min( retweet_values )

-    max_retweet_value = max( retweet_values )

+    if not len(lhs_times):

+        lhs_times = [time_posted,]

+    if not len(lhs_values):

+        lhs_values = [0,]

+    if not len(rhs_times):

+        rhs_times = [time_posted,]

+    if not len(rhs_values):

+        rhs_values = [0,]

+

+#    lhs_times = [(i - time_posted + 900) / 1800 for i in lhs_times]

+#    rhs_times = [(i - time_posted + 900) / 1800 for i in rhs_times]

+    lhs_times = [(i - time_posted) / 1800 for i in lhs_times]

+    rhs_times = [(i - time_posted) / 1800 for i in rhs_times]

+

+    min_comment_time = min(lhs_times)

+    max_comment_time = max(lhs_times)

+    min_comment_value = min(lhs_values)

+    max_comment_value = max(lhs_values)

+    min_rhs_time = min(rhs_times)

+    max_rhs_time = max(rhs_times)

+    min_rhs_value = min(rhs_values)

+    max_rhs_value = max(rhs_values)

     met_threshold_pt = -1

     if threshold_value != -1:

-        met_threshold_pt = index_id( retweet_values, operator.ge, threshold_value )

+        met_threshold_pt = index_id(rhs_values, operator.ge, threshold_value)

         if met_threshold_pt == -1 or tag_hit:

             # This can happen if threshold_value was set to a number

             # because the author or a tag was matched, but the article

@@ -182,30 +173,30 @@ def make_chart_url( time_posted, comment_times, comment_values, retweet_times,

         bg_color = odd_background

         watermark_color = odd_watermark

-    if len( comment_values ) < 8 and len( comment_values ) > 1:

+    if len(lhs_values) < 8 and len(lhs_values) > 1:

         # max_comment_value *= 2

         pass

-    elif len( comment_values ) == 1:

+    elif len(lhs_values) == 1:

         min_comment_value = 0

-    if len( retweet_values ) < 8 and len( retweet_values ) > 1:

-        # max_retweet_value *= 2

+    if len(rhs_values) < 8 and len(rhs_values) > 1:

+        # max_rhs_value *= 2

         pass

-    elif len( retweet_values ) == 1:

-        min_retweet_value = 0

+    elif len(rhs_values) == 1:

+        min_rhs_value = 0

     min_comment_value = 0

-    min_retweet_value = 0

+    min_rhs_value = 0

     chart_url = "http://chart.apis.google.com/chart?cht=lxy&chco=%s,%s&chs=%dx%d&chxs=0,%s|1,%s" % \

                 (series_1_color, series_2_color, img_width, img_height, series_1_color, series_2_color)

-    chart_url += "&chd=t:%s|%s|%s|%s" % ( ','.join( [ str( n ) for n in comment_times ] ),

-                                          ','.join( [ str( n ) for n in comment_values ] ),

-                                          ','.join( [ str( n ) for n in retweet_times ] ),

-                                          ','.join( [ str( n ) for n in retweet_values ] ) )

+    chart_url += "&chd=t:%s|%s|%s|%s" % (','.join([str(n) for n in lhs_times]),

+                                         ','.join([str(n) for n in lhs_values]),

+                                         ','.join([str(n) for n in rhs_times]),

+                                         ','.join([str(n) for n in rhs_values]))

     # TODO: Consider watermark levels, like:

     # chm=h,B0B0B0,1,0.3,1|r,E0E0E0,0,0,0.5

-    if max_retweet_value > 0:

-        threshold_percent = max( 0, min( (float(threshold_value) / max_retweet_value) - 0.01, 1.0  ) )

+    if max_rhs_value > 0:

+        threshold_percent = max(0, min((float(threshold_value) / max_rhs_value) - 0.01, 1.0))

     else:

         threshold_percent = 1.0

     chart_url += "&chm=r,%s,0,0,%1.3f" % (watermark_color, threshold_percent)

@@ -218,12 +209,12 @@ def make_chart_url( time_posted, comment_times, comment_values, retweet_times,

             dot_shape = 'o'

         chart_url += "|%s,%s,1,%d,10" % (dot_shape, dot_color, met_threshold_pt)

     chart_url += "&chxt=y,r&chxl=0:|%d|%d|1:|%d|%d&chds=%d,%d,%d,%d,%d,%d,%d,%d" % \

-                 ( min_comment_value, max_comment_value, min_retweet_value, max_retweet_value,

+                 (min_comment_value, max_comment_value, min_rhs_value, max_rhs_value,

                   0, max(7, max_comment_time),

                   min_comment_value, max_comment_value,

-                   0, max( 7, max_retweet_time ),

-                   min_comment_value, max_retweet_value )

-    chart_url += "&chf=bg,s,%s&chdl=comments|retweets" % ( bg_color, )

+                  0, max(7, max_rhs_time),

+                  min_comment_value, max_rhs_value)

+    chart_url += "&chf=bg,s,%s&chdl=comments|shares" % (bg_color,)

     return chart_url

@@ -276,8 +267,6 @@ def process_feed( yaml_items ):

             while len(yaml_items) > 200:

                 yaml_items.pop()

-#            cookie = Get_cookie( urllib2.Request( 'http://mediacdn.disqus.com/1078/build/system/count.js' ) )

-

             for i in yaml_items:

                 # i['title'] = asciiize(i['title'])

                 # i['tags'] = map(asciiize, i['tags'])

@@ -286,7 +275,7 @@ def process_feed( yaml_items ):

     else:

         if hasattr(feed, 'bozo_exception'):

             e = feed.bozo_exception

-            if isinstance( e, urllib2.URLError ): # e.__class__ == urllib2.URLError: # and hasattr(e, 'errno') and e.errno == 110:

+            if isinstance(e, urllib2.URLError):

                 print_last_line = True

                 if hasattr(e, 'reason'):

                     if e.reason[0] == 110:

@@ -354,12 +343,11 @@ def process_item( feed_item, yaml_items ):

                      'orig_posted'         : timecode_parsed,

                      'qualified'           : -1,

                      'comment_times'       : [],

-                      'comments'            : [],

+                     'fb_comments'         : [],

                      'fb_shares'           : [],

+                     'fb_likes'            : [],

                      'slash_comment_times' : [],

-                      'slash_comments'      : [],

-                      'retweet_times'       : [],

-                      'retweets'            : []

+                     'slash_comments'      : []

                     }

         if hasattr(feed_item, 'tags'):

             for i in feed_item.tags:

@@ -383,205 +371,53 @@ def process_yaml_item( yaml_item ):

     global any_entry_added

     timecode_now = int(time.time())

-    if len( yaml_item['comments'] ) < 8:

-        num_shares, num_comments = Get_fb_stats( yaml_item['link'] )

-#        disqus_id = Get_disqus_id( yaml_item )

-#        num_comments = Get_num_disqus_comments( yaml_item['link'], disqus_id, cookie )

+    if len(yaml_item['fb_comments']) < 8:

+        num_shares, num_comments, num_likes = Get_fb_stats(yaml_item['link'])

         if num_comments != -1:

             any_entry_added = True

             yaml_item['comment_times'].append(timecode_now)

-            yaml_item['comments'].append( num_comments )

             yaml_item['fb_shares'].append(num_shares)

+            yaml_item['fb_comments'].append(num_comments)

+            yaml_item['fb_likes'].append(num_likes)

-    if len( yaml_item['retweets'] ) < 8:

-        num_retweets = Get_num_retweets( yaml_item )

-        if num_retweets != -1:

-            any_entry_added = True

-            yaml_item['retweet_times'].append( timecode_now )

-            yaml_item['retweets'].append( num_retweets )

+#    if len(yaml_item['reddit_']) < 8:

+#        num_ = Get_reddit_stats(yaml_item['link'])

+#        if num_ != -1:

+#            any_entry_added = True

+#            yaml_item['reddit_times'].append(timecode_now)

+#            yaml_item['reddit_'].append(num_)

-def Get_num_comments( url_string ):

-    try:

-        f = urllib2.urlopen( url_string )

-        data = f.read()

-        f.close()

-    except urllib2.URLError, e:

-        if hasattr( e, 'reason' ):

-            print "Get_num_comments got an error:", e.reason

-        elif hasattr( e, 'code' ):

-            print "Get_num_comments got an error. Code:", e.code

-        return -1

-    tag_to_find = '<a href="#comments" rel="nofollow">'

-    offset = data.find( tag_to_find )

-    if offset != -1:

-        start_pos = offset + len( tag_to_find )

-        end_pos = start_pos

-        while str.isdigit( data[ end_pos ] ):

-            end_pos += 1

-        if end_pos > start_pos:

-            return int( data[start_pos:end_pos] )

+def Get_reddit_stats(url_string):

+    """ Consider curl "http://www.reddit.com/api/info.json?url=http://i.imgur.com/HG9dJ.jpg"

+    """

     return -1

-def Get_cookie( cookie_request ):

-    cookie = cookielib.CookieJar()

-    error_string = "Get_cookie didn't."

-    try:

-        cookie_response = urllib2.urlopen( cookie_request )

-        cookie.extract_cookies( cookie_response, cookie_request )

-        return cookie

-    except urllib2.URLError, e:

-        if hasattr( e, 'reason' ):

-            error_string = "Get_cookie got an error: %s" % ( str( e.reason ) )

-        elif hasattr( e, 'code' ):

-            error_string = "Get_cookie got an error. Code: %s" % ( str( e.code ) )

-    print error_string

-    return None

-

+def Get_fb_stats(url_string):

+    """ There are apparently two pretty good ways to do this. One, with FQL, querying for the parameters you want,

+    and two, with URL id. They go like this:

-def Get_disqus_id( yaml_item ):

-    if 'disqus_id' in yaml_item:

-        return yaml_item['disqus_id']

-    url_get_data = ''

-    try:

-        f = urllib2.urlopen( yaml_item['link'] )

-        data = f.read()

-        f.close()

-    except urllib2.URLError, e:

-        if hasattr( e, 'reason' ):

-            print "Get_disqus_id got an error:", e.reason

-        elif hasattr( e, 'code' ):

-            print "Get_disqus_id got an error. Code:", e.code, yaml_item['link']

-        return url_get_data

-    except httplib.BadStatusLine, e:

-        print "Get_discus_id got a BadStatusLine:", str( e )

-        return url_get_data

-

-    tag_to_find = '<a href="#comments" rel="nofollow"><span class="dsq-postid" rel="'

-    offset = data.find( tag_to_find )

-    if offset != -1:

-        start_pos = offset + len( tag_to_find )

-        end_pos = start_pos

-        while data[ end_pos ] != '"' and end_pos < start_pos + 200:

-            end_pos += 1

-        if end_pos < start_pos + 200:

-            url_get_data = urllib.quote_plus( data[start_pos:end_pos] ).replace( '+', '%20' )

-            yaml_item['disqus_id'] = url_get_data

-#    else:

-#        print "Get_disqus_id could not find #comments anchor for", yaml_item['link']

-    return url_get_data

-

-

-def Get_num_disqus_comments( url_string, disqus_id, cookie ):

-

-    if cookie == None or disqus_id == '':

-        return -1

+    FQL:

-    opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cookie ) )

-    request = urllib2.Request( 'http://disqus.com/forums/techcrunch/count.js?q=1&0=1,' + disqus_id )

-    try:

-        response = opener.open( request )

-        disqus_data = response.read()

-    except urllib2.URLError, e:

-        if hasattr( e, 'reason' ):

-            print "Get_num_disqus_comments got an error getting the count:", e.reason

-        elif hasattr( e, 'code' ):

-            print "Get_num_disqus_comments got an error getting the count. Code:", e.code

-        return -1

-    disqus_tag_to_find = 'displayCount('

-    disqus_offset = disqus_data.find( disqus_tag_to_find )

-    if disqus_offset != -1:

-        start_pos = disqus_offset + len( disqus_tag_to_find )

-        end_pos = disqus_data.find( '}]})', start_pos )

-        if end_pos != -1:

-            return int( json.loads( disqus_data[start_pos:end_pos+3] )['counts'][0]['comments'] )

-    else:

-        print "Get_num_disqus_comments found no disqus tag for", url_string

-    return -1

+    u = urllib.quote_plus(url_string)

+    urllib2.urlopen('https://graph.facebook.com/fql?q=SELECT%%20total_count,comment_count,like_count,share_count%%20FROM%%20link_stat%%20WHERE%%20url=%%27%s%%27' % (u))

+    URL ID:

-def Get_num_retweets_unused( yaml_item ):

-    """ TODO: Support for retweents has been removed.

-    See: https://twittercommunity.com/t/a-new-design-for-tweet-and-follow-buttons/52791

-    So instead, use facebook.

-    curl https://graph.facebook.com/fql?q=SELECT%20total_count,comment_count,like_count,share_count%20FROM%20link_stat%20WHERE%20url=%27http://techcrunch.com/2015/11/22/the-real-reason-on-demand-startups-are-reclassifying-workers/?ncid=rss%27

+    u = urllib.quote_plus(url_string)

+    with open('/home/dblume/oauth.dlma.com/facebook-token.txt', 'r') as f:

+        token = f.read()

+    encoded = urllib.urlencode({'access_token': token})

+    urllib2.urlopen('https://graph.facebook.com/v2.5/?id=%s&%s' % (u, encoded)

     """

-    url_string = yaml_item['link']

-    try:

-        f = urllib2.urlopen( 'http://api.tweetmeme.com/button.js?url=%s' % ( url_string ) )

-        data = f.read()

-        f.close()

-    except urllib2.URLError, e:

-        if hasattr( e, 'reason' ):

-            print "Get_num_retweets got an error:", e.reason

-        elif hasattr( e, 'code' ):

-            print "Get_num_retweets got an error. Code:", e.code

-        return -1

-    tag_to_find = '<span class="c">'

-    offset = data.find( tag_to_find )

-    if offset != -1:

-        start_pos = offset + len( tag_to_find )

-        end_pos = data.find( '<', start_pos )

-        if end_pos != -1:

-            try:

-                return int( data[ start_pos:end_pos ] )

-            except ValueError, e:

-                if data[ start_pos:end_pos ] != '?':

-                    print "Get_num_retweets expected a number but got \"%s\"" % ( data[ start_pos:end_pos ], )

-                else:

-                    print "Get_num_retweets got '?' for \"%s...\", posted %s ago." % \

-                          ( yaml_item['title'][:20],

-                            texttime.stringify( timedelta( seconds = time.time() - yaml_item['orig_posted'] ) )

-                          )

-    return -1

-

-

-def Get_num_retweets( yaml_item ):

-    """ TODO: Support for retweents has been removed.

-    See: https://twittercommunity.com/t/a-new-design-for-tweet-and-follow-buttons/52791

-    So instead, use facebook.

-    curl https://graph.facebook.com/fql?q=SELECT%20total_count,comment_count,like_count,share_count%20FROM%20link_stat%20WHERE%20url=%27http://techcrunch.com/2015/11/22/the-real-reason-on-demand-startups-are-reclassifying-workers/?ncid=rss%27

-    """

-    url_string = yaml_item['link']

-    try:

-        f = urllib2.urlopen( 'http://urls.api.twitter.com/1/urls/count.json?url=%s&callback=twttr.receiveCount' % \

-                             urllib.quote_plus( url_string ) )

-        data = f.read()

-        f.close()

-    except (urllib2.URLError, httplib.BadStatusLine), e:

-        if hasattr( e, 'reason' ):

-            print "Get_num_retweets got an error:", e.reason

-        elif hasattr( e, 'code' ):

-            print "Get_num_retweets got an error. Code:", e.code

-        else:

-            print "Get_num_retweets got an error:", str( e )

-        return -1

-    tag_to_find = '"count":'

-    offset = data.find( tag_to_find )

-    if offset != -1:

-        start_pos = offset + len( tag_to_find )

-        end_pos = data.find( ',', start_pos )

-        if end_pos != -1:

-            try:

-                return int( data[ start_pos:end_pos ] )

-            except ValueError, e:

-                if data[ start_pos:end_pos ] != '?':

-                    print "Get_num_retweets expected a number but got \"%s\"" % ( data[ start_pos:end_pos ], )

-                else:

-                    print "Get_num_retweets got '?' for \"%s...\", posted %s ago." % \

-                          ( yaml_item['title'][:20],

-                            texttime.stringify( timedelta( seconds = time.time() - yaml_item['orig_posted'] ) )

-                          )

-    return -1

-

-

-def Get_fb_stats( url_string ):

-    """ Returns shares and comments """

     shares = -1

     comments = -1

+    likes = -1

+

     try:

-        f = urllib2.urlopen( 'https://graph.facebook.com/?ids=' + url_string )

+        url = 'https://graph.facebook.com/fql?q=SELECT%%20total_count,comment_count,like_count,share_count%%20FROM%%20link_stat%%20WHERE%%20url=%%27%s%%27'

+        f = urllib2.urlopen(url % (urllib.quote_plus(url_string)))

         data = f.read()

         f.close()

     except (urllib2.URLError, httplib.BadStatusLine), e:

@@ -591,20 +427,24 @@ def Get_fb_stats( url_string ):

             print "Get_fb_stats got an error. Code:", e.code, url_string

         else:

             print "Get_fb_stats got an error:", str(e)

-        return -1, -1

-    if len( data ) > len( url_string ):

-        d = json.loads( data ).values()[0]

-        if d.has_key( 'shares' ):

-            shares = d['shares']

+        return shares, comments, likes

+    if len(data) > 20:

+        d = json.loads(data)['data'][0]

+        if 'like_count' in d:

+            likes = d['like_count']

         else:

-            shares = 0

-        if d.has_key( 'comments' ):

-            comments = d['comments']

+            likes = 0

+        if 'comment_count' in d:

+            comments = d['comment_count']

         else:

             comments = 0

+        if 'share_count' in d:

+            shares = d['share_count']

+        else:

+            shares = 0

     else:

         print "Get_fb_stats got too little data for ",  url_string

-    return shares, comments

+    return shares, comments, likes

 def Save_image(url_string, file_path):

@@ -655,15 +495,15 @@ def Make_index_html( yaml_items, weekend_stats, weekday_stats ):

             tag_hit = True

         chart_url = make_chart_url(image['orig_posted'],

                                    image['comment_times'],

-                                    image['comments'],

-                                    image['retweet_times'],

-                                    image['retweets'],

+                                   image['fb_comments'],

+                                   image[rhs_metric_times],

+                                   image[rhs_metric],

                                    image['qualified'],

                                    image_index % 2,

                                    tag_hit

                                   )

 #        if image['title'].startswith( 'Too ' ):

-#            print image['title'], image['qualified'], image['retweet_times']

+#            print image['title'], image['qualified'], image['rhs_times']

 #            print chart_url

         image_url = Save_image(chart_url, os.path.join(cache_path, '%d_%d.png' % (cur_time, image_index)))

         f.write('<tr valign="center" class="%s">\n  <td><strong><a href="%s">%s</a></strong> <span class="author">by %s</span></td>\n' % \

@@ -727,12 +567,11 @@ if __name__=='__main__':

         #     'tags'                : [ u'Google', u'privacy' ]

         #     'qualified'           : -1

         #     'comment_times'       : [ 1282197199, 1282197407 ]

-        #     'comments'            : [ 0, 15 ]

-        #     'fb_shares'           : [ 0, 3 ]

+        #     'fb_comments'         : [ 0, 5 ]

+        #     'fb_shares'           : [ 0, 300 ]

+        #     'fb_likes'            : [ 0, 19 ]

         #     'slash_comment_times' : [ 1282197199, 1282197407 ]

         #     'slash_comments'      : [ 0, 5 ]

-        #     'retweet_times'       : [ 1282197199, 1282197407 ]

-        #     'retweets'            : [ 0, 43 ]

         #    },

         #    { ... }

         #  ]

@@ -757,8 +596,7 @@ if __name__=='__main__':

         # If any work was done, then write files.

         #

         if any_entry_added:

-

-            weekend_stats, weekday_stats = analysis.Process_retweets_for_feed( items )

+            weekend_stats, weekday_stats = analysis.Process_feed(items, rhs_metric, rhs_metric_times)

             # We'll only look at the stats for the time 1:00 to 1:30 after posting.

             weekend_median, weekend_mean, weekend_sigma = weekend_stats[2]

@@ -772,17 +610,17 @@ if __name__=='__main__':

                 else:

                     threshold = weekday_threshold

                 if item['qualified'] == -1:

-                    for i in range( len( item['retweet_times'] ) ):

-                        r_time = item['retweet_times'][i]

+                    for i in range(len(item[rhs_metric_times])):

+                        r_time = item[rhs_metric_times][i]

                         if r_time - item['orig_posted'] < 5400:

-                            if item['retweets'][i] >= threshold:

+                            if item[rhs_metric][i] >= threshold:

                                 item['qualified'] = threshold

                             if r_time - item['orig_posted'] >= 3600:

                                 break

             # Automatically add those items whose authors and tags I like

             for item in items:

-                if item['qualified'] == -1 and len( item['retweet_times'] ) > 0:

+                if item['qualified'] == -1 and len(item[rhs_metric_times]) > 0:

                     if item['author'].lower() in authors_to_post:

                         item['qualified'] = threshold

                     elif len(set([j.lower() for j in item['tags']]) & tags_to_post) > 0: