techcrunch.git

@@ -4,8 +4,10 @@

 # cp techcrunch.yaml techcrunch.yaml_back; ./techcrunch.py; \

 # cp techcrunch.yaml techcrunch_tmp.yaml; cp techcrunch.yaml_back techcrunch.yaml

 import feedparser

-import yaml

 import sys

+if sys.path[0] != '':

+    sys.path.insert(0, '')  # DXB Only needed for cronjobs to find yaml

+import yaml

 import os

 import time

 import codecs

@@ -31,6 +33,9 @@ any_entry_added = False

 tags_to_post = {'apple', 'google', 'roku'}

 authors_to_post = ['michael arrington',]

+# 2022-02-24: Seeing too many posts with just mean + sigma, adjust with factor

+threshold_sigma_factor = 1.5

+

 # TODO 2018-01-18: Maybe combine fb_likes with bf_shares or something...

 rhs_metric = 'fb_likes'

 rhs_metric_times = 'comment_times'

@@ -206,7 +211,8 @@ def process_feed(yaml_items):

     """Retrieve the url and process it.

     feed_info (in, out) A tuple that describes an individual feed, like its name and etag.

     """

-    feed = feedparser.parse('https://techcrunch.com/feed/')

+    #feed = feedparser.parse('https://techcrunch.com/feed/')

+    feed = feedparser.parse('https://pi.dlma.com/techcrunch_feed.php')  # DXB temporary until removed from denylist

     if hasattr(feed, 'status'):

         if feed.status == 304:

             pass

@@ -460,9 +466,9 @@ def make_index_html(yaml_items, weekend_stats, weekday_stats):

     with codecs.open(new_index_fullpath, 'w', 'utf-8') as f:

         f.write(html_head % (even_background, odd_background, img_width, chart_io.getvalue()))

         chart_io.close()

-        f.write('<div align="center">\n<table class="legend">\n<tr><th></th><th>Median</th><th>Mean</th><th>Std. Dev</th><th>Threshold</th></tr>\n')

-        f.write('<tr><th>Weekday</th><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td></tr>\n' % (weekday_stats[2][0], weekday_stats[2][1], weekday_stats[2][2], weekday_stats[2][1] + weekday_stats[2][2]))

-        f.write('<tr><th>Weekend</th><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td></tr>\n' % (weekend_stats[2][0], weekend_stats[2][1], weekend_stats[2][2], weekend_stats[2][1] + weekend_stats[2][2]))

+        f.write('<div align="center">\n<table class="legend">\n<tr><th></th><th>Median</th><th>Mean</th><th>Std. Dev</th><th>Threshold (mean + sigma * %1.1f)</th></tr>\n' % threshold_sigma_factor)

+        f.write('<tr><th>Weekday</th><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td></tr>\n' % (weekday_stats[2][0], weekday_stats[2][1], weekday_stats[2][2], weekday_stats[2][1] + weekday_stats[2][2] * threshold_sigma_factor))

+        f.write('<tr><th>Weekend</th><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td><td>%1.1f</td></tr>\n' % (weekend_stats[2][0], weekend_stats[2][1], weekend_stats[2][2], weekend_stats[2][1] + weekend_stats[2][2] * threshold_sigma_factor))

         f.write('</table></div>\n<br />\n')

         f.write('<div align="center">\n<table>\n')

         for image_index, image in enumerate(yaml_items[:40]):

@@ -534,7 +540,7 @@ if __name__=='__main__':

         yaml_fullpath = os.path.join(localdir, 'techcrunch.yaml')

         if os.path.exists(yaml_fullpath):

             with open(yaml_fullpath, 'rb') as f:

-                items = yaml.load(f)

+                items = yaml.load(f, Loader=yaml.Loader)

                 if items is None:

                     print yaml_fullpath, "exists, but was empty."

                     items = []

@@ -562,9 +568,9 @@ if __name__=='__main__':

             # We'll only look at the stats up to 2 hours after posting.

             weekend_median, weekend_mean, weekend_sigma = weekend_stats[2]

-            weekend_threshold = weekend_mean + weekend_sigma

+            weekend_threshold = weekend_mean + weekend_sigma * threshold_sigma_factor

             weekday_median, weekday_mean, weekday_sigma = weekday_stats[2]

-            weekday_threshold = weekday_mean + weekday_sigma

+            weekday_threshold = weekday_mean + weekday_sigma * threshold_sigma_factor

             for item in items:

                 wday = time.localtime(item['orig_posted']).tm_wday

                 if wday == 5 or wday == 6:

@@ -597,15 +603,15 @@ if __name__=='__main__':

             # For the one file we really use, write to a file on the side, then move it.

             yaml_newfile_fullpath = os.path.join(localdir, 'techcrunch_temp_writable.yaml')

             with open(yaml_newfile_fullpath, 'wb') as f:

-                yaml.dump(items, f, width=120)

+                yaml.dump(items, f, default_flow_style=None, width=120)

             try:

                 os.rename(yaml_newfile_fullpath, yaml_fullpath)

             except OSError as e:

                 print "The source file was", yaml_newfile_fullpath, "and exists =", os.path.isfile(yaml_newfile_fullpath)

             with open(os.path.join(localdir, 'techcrunch_text.yaml'), 'w') as f:

-                yaml.dump(items, f, width=120)

+                yaml.dump(items, f, default_flow_style=None, width=120)

             with codecs.open(os.path.join(localdir, 'techcrunch_unicode.yaml'), 'w', 'utf-8') as f:

-                yaml.dump(items, f, encoding='utf-8', width=120)

+                yaml.dump(items, f, default_flow_style=None, encoding='utf-8', width=120)

             make_feed_file(items)