techcrunch.git

@@ -0,0 +1,19 @@

+Copyright (c) 2018, David Blume

+

+Permission is hereby granted, free of charge, to any person obtaining

+a copy of this software and associated documentation files (the "Software"),

+to deal in the Software without restriction, including without limitation

+the rights to use, copy, modify, merge, publish, distribute, sublicense,

+and/or sell copies of the Software, and to permit persons to whom the

+Software is furnished to do so, subject to the following conditions:

+

+The above copyright notice and this permission notice shall be included

+in all copies or substantial portions of the Software.

+

+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS

+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER

+DEALINGS IN THE SOFTWARE.

@@ -0,0 +1,30 @@

+[![License](https://img.shields.io/badge/license-MIT_license-blue.svg)](http://git.dlma.com/techcrunch.git/blob/master/LICENSE.txt)

+![python2.x](https://img.shields.io/badge/python-2.x-yellow.svg)

+# TechCrunch Feed Filter

+

+This is a Python script run as a cronjob to read the TechCrunch article feed, 

+and decide which articles to include in its own feed.

+

+Here's a [blog post about it](http://david.dlma.com/blog/my-techcrunch-feed-filter).

+

+# History

+

+This was originally archived in a Subversion repo. I'd forgotten about the

+version control and had gotten into the habit of just modifying the production

+site.

+

+* 2010-09-03: Original

+* 2010-09-03: Save off the disqus identifier for use later.

+* 2011-02-04: Algorithm changes (tags and author checked), new chart drawing, spaces used instead of tabs.

+* 2011-02-04: Update to the chart drawing algorithm.

+* 2013-08-04: Miscellaneous changes to techcrunch.py

+* 2015-11-23: Resync svn with production site.

+* 2015-11-27: Remove obsolete disqus and retweet code, and refactor style to be more PEP-8ish.

+

+# Is it any good?

+

+[Yes](https://news.ycombinator.com/item?id=3067434).

+

+# Licence

+

+This software uses the [MIT license](http://git.dlma.com/techcrunch.git/blob/master/LICENSE.txt)

@@ -0,0 +1,3612 @@

+#!/usr/bin/env python

+"""Universal feed parser

+

+Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds

+

+Visit http://feedparser.org/ for the latest version

+Visit http://feedparser.org/docs/ for the latest documentation

+

+Required: Python 2.1 or later

+Recommended: Python 2.3 or later

+Recommended: CJKCodecs and iconv_codec <http://cjkpython.i18n.org/>

+"""

+

+__version__ = "4.2-pre-" + "$Revision: 291 $"[11:14] + "-svn"

+__license__ = """Copyright (c) 2002-2008, Mark Pilgrim, All rights reserved.

+

+Redistribution and use in source and binary forms, with or without modification,

+are permitted provided that the following conditions are met:

+

+* Redistributions of source code must retain the above copyright notice,

+  this list of conditions and the following disclaimer.

+* Redistributions in binary form must reproduce the above copyright notice,

+  this list of conditions and the following disclaimer in the documentation

+  and/or other materials provided with the distribution.

+

+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'

+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE

+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

+POSSIBILITY OF SUCH DAMAGE."""

+__author__ = "Mark Pilgrim <http://diveintomark.org/>"

+__contributors__ = ["Jason Diamond <http://injektilo.org/>",

+                    "John Beimler <http://john.beimler.org/>",

+                    "Fazal Majid <http://www.majid.info/mylos/weblog/>",

+                    "Aaron Swartz <http://aaronsw.com/>",

+                    "Kevin Marks <http://epeus.blogspot.com/>",

+                    "Sam Ruby <http://intertwingly.net/>"]

+_debug = 0

+

+# HTTP "User-Agent" header to send to servers when downloading feeds.

+# If you are embedding feedparser in a larger application, you should

+# change this to your application name and URL.

+USER_AGENT = "UniversalFeedParser/%s +http://feedparser.org/" % __version__

+

+# HTTP "Accept" header to send to servers when downloading feeds.  If you don't

+# want to send an Accept header, set this to None.

+ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1"

+

+# List of preferred XML parsers, by SAX driver name.  These will be tried first,

+# but if they're not installed, Python will keep searching through its own list

+# of pre-installed parsers until it finds one that supports everything we need.

+PREFERRED_XML_PARSERS = ["drv_libxml2"]

+

+# If you want feedparser to automatically run HTML markup through HTML Tidy, set

+# this to 1.  Requires mxTidy <http://www.egenix.com/files/python/mxTidy.html>

+# or utidylib <http://utidylib.berlios.de/>.

+TIDY_MARKUP = 0

+

+# List of Python interfaces for HTML Tidy, in order of preference.  Only useful

+# if TIDY_MARKUP = 1

+PREFERRED_TIDY_INTERFACES = ["uTidy", "mxTidy"]

+

+# If you want feedparser to automatically resolve all relative URIs, set this

+# to 1.

+RESOLVE_RELATIVE_URIS = 1

+

+# If you want feedparser to automatically sanitize all potentially unsafe

+# HTML content, set this to 1.

+SANITIZE_HTML = 1

+

+# ---------- required modules (should come with any Python distribution) ----------

+import sgmllib, re, sys, copy, urlparse, time, rfc822, types, cgi, urllib, urllib2

+try:

+    from cStringIO import StringIO as _StringIO

+except:

+    from StringIO import StringIO as _StringIO

+

+# ---------- optional modules (feedparser will work without these, but with reduced functionality) ----------

+

+# gzip is included with most Python distributions, but may not be available if you compiled your own

+try:

+    import gzip

+except:

+    gzip = None

+try:

+    import zlib

+except:

+    zlib = None

+

+# If a real XML parser is available, feedparser will attempt to use it.  feedparser has

+# been tested with the built-in SAX parser, PyXML, and libxml2.  On platforms where the

+# Python distribution does not come with an XML parser (such as Mac OS X 10.2 and some

+# versions of FreeBSD), feedparser will quietly fall back on regex-based parsing.

+try:

+    import xml.sax

+    xml.sax.make_parser(PREFERRED_XML_PARSERS) # test for valid parsers

+    from xml.sax.saxutils import escape as _xmlescape

+    _XML_AVAILABLE = 1

+except:

+    _XML_AVAILABLE = 0

+    def _xmlescape(data,entities={}):

+        data = data.replace('&', '&')

+        data = data.replace('>', '>')

+        data = data.replace('<', '&lt;')

+        for char, entity in entities:

+            data = data.replace(char, entity)

+        return data

+

+# base64 support for Atom feeds that contain embedded binary data

+try:

+    import base64, binascii

+except:

+    base64 = binascii = None

+

+# cjkcodecs and iconv_codec provide support for more character encodings.

+# Both are available from http://cjkpython.i18n.org/

+try:

+    import cjkcodecs.aliases

+except:

+    pass

+try:

+    import iconv_codec

+except:

+    pass

+

+# chardet library auto-detects character encodings

+# Download from http://chardet.feedparser.org/

+try:

+    import chardet

+    if _debug:

+        import chardet.constants

+        chardet.constants._debug = 1

+except:

+    chardet = None

+

+# reversable htmlentitydefs mappings for Python 2.2

+try:

+  from htmlentitydefs import name2codepoint, codepoint2name

+except:

+  import htmlentitydefs

+  name2codepoint={}

+  codepoint2name={}

+  for (name,codepoint) in htmlentitydefs.entitydefs.iteritems():

+    if codepoint.startswith('&#'): codepoint=unichr(int(codepoint[2:-1]))

+    name2codepoint[name]=ord(codepoint)

+    codepoint2name[ord(codepoint)]=name

+

+# BeautifulSoup parser used for parsing microformats from embedded HTML content

+# http://www.crummy.com/software/BeautifulSoup/

+# feedparser is tested with BeautifulSoup 3.0.x, but it might work with the

+# older 2.x series.  If it doesn't, and you can figure out why, I'll accept a

+# patch and modify the compatibility statement accordingly.

+try:

+    import BeautifulSoup

+except:

+    BeautifulSoup = None

+

+# ---------- don't touch these ----------

+class ThingsNobodyCaresAboutButMe(Exception): pass

+class CharacterEncodingOverride(ThingsNobodyCaresAboutButMe): pass

+class CharacterEncodingUnknown(ThingsNobodyCaresAboutButMe): pass

+class NonXMLContentType(ThingsNobodyCaresAboutButMe): pass

+class UndeclaredNamespace(Exception): pass

+

+sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')

+sgmllib.special = re.compile('<!')

+sgmllib.charref = re.compile('&#(\d+|x[0-9a-fA-F]+);')

+

+if sgmllib.endbracket.search(' <').start(0):

+    class EndBracketMatch:

+        endbracket = re.compile('''([^'"<>]|"[^"]*"(?=>|/|\s|\w+=)|'[^']*'(?=>|/|\s|\w+=))*(?=[<>])|.*?(?=[<>])''')

+        def search(self,string,index=0):

+            self.match = self.endbracket.match(string,index)

+            if self.match: return self

+        def start(self,n):

+            return self.match.end(n)

+    sgmllib.endbracket = EndBracketMatch()

+

+SUPPORTED_VERSIONS = {'': 'unknown',

+                      'rss090': 'RSS 0.90',

+                      'rss091n': 'RSS 0.91 (Netscape)',

+                      'rss091u': 'RSS 0.91 (Userland)',

+                      'rss092': 'RSS 0.92',

+                      'rss093': 'RSS 0.93',

+                      'rss094': 'RSS 0.94',

+                      'rss20': 'RSS 2.0',

+                      'rss10': 'RSS 1.0',

+                      'rss': 'RSS (unknown version)',

+                      'atom01': 'Atom 0.1',

+                      'atom02': 'Atom 0.2',

+                      'atom03': 'Atom 0.3',

+                      'atom10': 'Atom 1.0',

+                      'atom': 'Atom (unknown version)',

+                      'cdf': 'CDF',

+                      'hotrss': 'Hot RSS'

+                      }

+

+try:

+    UserDict = dict

+except NameError:

+    # Python 2.1 does not have dict

+    from UserDict import UserDict

+    def dict(aList):

+        rc = {}

+        for k, v in aList:

+            rc[k] = v

+        return rc

+

+class FeedParserDict(UserDict):

+    keymap = {'channel': 'feed',

+              'items': 'entries',

+              'guid': 'id',

+              'date': 'updated',

+              'date_parsed': 'updated_parsed',

+              'description': ['subtitle', 'summary'],

+              'url': ['href'],

+              'modified': 'updated',

+              'modified_parsed': 'updated_parsed',

+              'issued': 'published',

+              'issued_parsed': 'published_parsed',

+              'copyright': 'rights',

+              'copyright_detail': 'rights_detail',

+              'tagline': 'subtitle',

+              'tagline_detail': 'subtitle_detail'}

+    def __getitem__(self, key):

+        if key == 'category':

+            return UserDict.__getitem__(self, 'tags')[0]['term']

+        if key == 'enclosures':

+            norel = lambda link: FeedParserDict([(name,value) for (name,value) in link.items() if name!='rel'])

+            return [norel(link) for link in UserDict.__getitem__(self, 'links') if link['rel']=='enclosure']

+        if key == 'license':

+            for link in UserDict.__getitem__(self, 'links'):

+                if link['rel']=='license' and link.has_key('href'):

+                    return link['href']

+        if key == 'categories':

+            return [(tag['scheme'], tag['term']) for tag in UserDict.__getitem__(self, 'tags')]

+        realkey = self.keymap.get(key, key)

+        if type(realkey) == types.ListType:

+            for k in realkey:

+                if UserDict.has_key(self, k):

+                    return UserDict.__getitem__(self, k)

+        if UserDict.has_key(self, key):

+            return UserDict.__getitem__(self, key)

+        return UserDict.__getitem__(self, realkey)

+

+    def __setitem__(self, key, value):

+        for k in self.keymap.keys():

+            if key == k:

+                key = self.keymap[k]

+                if type(key) == types.ListType:

+                    key = key[0]

+        return UserDict.__setitem__(self, key, value)

+

+    def get(self, key, default=None):

+        if self.has_key(key):

+            return self[key]

+        else:

+            return default

+

+    def setdefault(self, key, value):

+        if not self.has_key(key):

+            self[key] = value

+        return self[key]

+

+    def has_key(self, key):

+        try:

+            return hasattr(self, key) or UserDict.has_key(self, key)

+        except AttributeError:

+            return False

+

+    def __getattr__(self, key):

+        try:

+            return self.__dict__[key]

+        except KeyError:

+            pass

+        try:

+            assert not key.startswith('_')

+            return self.__getitem__(key)

+        except:

+            raise AttributeError, "object has no attribute '%s'" % key

+

+    def __setattr__(self, key, value):

+        if key.startswith('_') or key == 'data':

+            self.__dict__[key] = value

+        else:

+            return self.__setitem__(key, value)

+

+    def __contains__(self, key):

+        return self.has_key(key)

+

+def zopeCompatibilityHack():

+    global FeedParserDict

+    del FeedParserDict

+    def FeedParserDict(aDict=None):

+        rc = {}

+        if aDict:

+            rc.update(aDict)

+        return rc

+

+_ebcdic_to_ascii_map = None

+def _ebcdic_to_ascii(s):

+    global _ebcdic_to_ascii_map

+    if not _ebcdic_to_ascii_map:

+        emap = (

+            0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,

+            16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,

+            128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,

+            144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,

+            32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,

+            38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,

+            45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,

+            186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,

+            195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,201,

+            202,106,107,108,109,110,111,112,113,114,203,204,205,206,207,208,

+            209,126,115,116,117,118,119,120,121,122,210,211,212,213,214,215,

+            216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,

+            123,65,66,67,68,69,70,71,72,73,232,233,234,235,236,237,

+            125,74,75,76,77,78,79,80,81,82,238,239,240,241,242,243,

+            92,159,83,84,85,86,87,88,89,90,244,245,246,247,248,249,

+            48,49,50,51,52,53,54,55,56,57,250,251,252,253,254,255

+            )

+        import string

+        _ebcdic_to_ascii_map = string.maketrans( \

+            ''.join(map(chr, range(256))), ''.join(map(chr, emap)))

+    return s.translate(_ebcdic_to_ascii_map)

+

+_cp1252 = {

+  unichr(128): unichr(8364), # euro sign

+  unichr(130): unichr(8218), # single low-9 quotation mark

+  unichr(131): unichr( 402), # latin small letter f with hook

+  unichr(132): unichr(8222), # double low-9 quotation mark

+  unichr(133): unichr(8230), # horizontal ellipsis

+  unichr(134): unichr(8224), # dagger

+  unichr(135): unichr(8225), # double dagger

+  unichr(136): unichr( 710), # modifier letter circumflex accent

+  unichr(137): unichr(8240), # per mille sign

+  unichr(138): unichr( 352), # latin capital letter s with caron

+  unichr(139): unichr(8249), # single left-pointing angle quotation mark

+  unichr(140): unichr( 338), # latin capital ligature oe

+  unichr(142): unichr( 381), # latin capital letter z with caron

+  unichr(145): unichr(8216), # left single quotation mark

+  unichr(146): unichr(8217), # right single quotation mark

+  unichr(147): unichr(8220), # left double quotation mark

+  unichr(148): unichr(8221), # right double quotation mark

+  unichr(149): unichr(8226), # bullet

+  unichr(150): unichr(8211), # en dash

+  unichr(151): unichr(8212), # em dash

+  unichr(152): unichr( 732), # small tilde

+  unichr(153): unichr(8482), # trade mark sign

+  unichr(154): unichr( 353), # latin small letter s with caron

+  unichr(155): unichr(8250), # single right-pointing angle quotation mark

+  unichr(156): unichr( 339), # latin small ligature oe

+  unichr(158): unichr( 382), # latin small letter z with caron

+  unichr(159): unichr( 376)} # latin capital letter y with diaeresis

+

+_urifixer = re.compile('^([A-Za-z][A-Za-z0-9+-.]*://)(/*)(.*?)')

+def _urljoin(base, uri):

+    uri = _urifixer.sub(r'\1\3', uri)

+    try:

+        return urlparse.urljoin(base, uri)

+    except:

+        uri = urlparse.urlunparse([urllib.quote(part) for part in urlparse.urlparse(uri)])

+        return urlparse.urljoin(base, uri)

+

+class _FeedParserMixin:

+    namespaces = {'': '',

+                  'http://backend.userland.com/rss': '',

+                  'http://blogs.law.harvard.edu/tech/rss': '',

+                  'http://purl.org/rss/1.0/': '',

+                  'http://my.netscape.com/rdf/simple/0.9/': '',

+                  'http://example.com/newformat#': '',

+                  'http://example.com/necho': '',

+                  'http://purl.org/echo/': '',

+                  'uri/of/echo/namespace#': '',

+                  'http://purl.org/pie/': '',

+                  'http://purl.org/atom/ns#': '',

+                  'http://www.w3.org/2005/Atom': '',

+                  'http://purl.org/rss/1.0/modules/rss091#': '',

+

+                  'http://webns.net/mvcb/':                               'admin',

+                  'http://purl.org/rss/1.0/modules/aggregation/':         'ag',

+                  'http://purl.org/rss/1.0/modules/annotate/':            'annotate',

+                  'http://media.tangent.org/rss/1.0/':                    'audio',

+                  'http://backend.userland.com/blogChannelModule':        'blogChannel',

+                  'http://web.resource.org/cc/':                          'cc',

+                  'http://backend.userland.com/creativeCommonsRssModule': 'creativeCommons',

+                  'http://purl.org/rss/1.0/modules/company':              'co',

+                  'http://purl.org/rss/1.0/modules/content/':             'content',

+                  'http://my.theinfo.org/changed/1.0/rss/':               'cp',

+                  'http://purl.org/dc/elements/1.1/':                     'dc',

+                  'http://purl.org/dc/terms/':                            'dcterms',

+                  'http://purl.org/rss/1.0/modules/email/':               'email',

+                  'http://purl.org/rss/1.0/modules/event/':               'ev',

+                  'http://rssnamespace.org/feedburner/ext/1.0':           'feedburner',

+                  'http://freshmeat.net/rss/fm/':                         'fm',

+                  'http://xmlns.com/foaf/0.1/':                           'foaf',

+                  'http://www.w3.org/2003/01/geo/wgs84_pos#':             'geo',

+                  'http://postneo.com/icbm/':                             'icbm',

+                  'http://purl.org/rss/1.0/modules/image/':               'image',

+                  'http://www.itunes.com/DTDs/PodCast-1.0.dtd':           'itunes',

+                  'http://example.com/DTDs/PodCast-1.0.dtd':              'itunes',

+                  'http://purl.org/rss/1.0/modules/link/':                'l',

+                  'http://search.yahoo.com/mrss':                         'media',

+                  'http://madskills.com/public/xml/rss/module/pingback/': 'pingback',

+                  'http://prismstandard.org/namespaces/1.2/basic/':       'prism',

+                  'http://www.w3.org/1999/02/22-rdf-syntax-ns#':          'rdf',

+                  'http://www.w3.org/2000/01/rdf-schema#':                'rdfs',

+                  'http://purl.org/rss/1.0/modules/reference/':           'ref',

+                  'http://purl.org/rss/1.0/modules/richequiv/':           'reqv',

+                  'http://purl.org/rss/1.0/modules/search/':              'search',

+                  'http://purl.org/rss/1.0/modules/slash/':               'slash',

+                  'http://schemas.xmlsoap.org/soap/envelope/':            'soap',

+                  'http://purl.org/rss/1.0/modules/servicestatus/':       'ss',

+                  'http://hacks.benhammersley.com/rss/streaming/':        'str',

+                  'http://purl.org/rss/1.0/modules/subscription/':        'sub',

+                  'http://purl.org/rss/1.0/modules/syndication/':         'sy',

+                  'http://schemas.pocketsoap.com/rss/myDescModule/':      'szf',

+                  'http://purl.org/rss/1.0/modules/taxonomy/':            'taxo',

+                  'http://purl.org/rss/1.0/modules/threading/':           'thr',

+                  'http://purl.org/rss/1.0/modules/textinput/':           'ti',

+                  'http://madskills.com/public/xml/rss/module/trackback/':'trackback',

+                  'http://wellformedweb.org/commentAPI/':                 'wfw',

+                  'http://purl.org/rss/1.0/modules/wiki/':                'wiki',

+                  'http://www.w3.org/1999/xhtml':                         'xhtml',

+                  'http://www.w3.org/1999/xlink':                         'xlink',

+                  'http://www.w3.org/XML/1998/namespace':                 'xml'

+}

+    _matchnamespaces = {}

+

+    can_be_relative_uri = ['link', 'id', 'wfw_comment', 'wfw_commentrss', 'docs', 'url', 'href', 'comments', 'icon', 'logo']

+    can_contain_relative_uris = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description']

+    can_contain_dangerous_markup = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description']

+    html_types = ['text/html', 'application/xhtml+xml']

+

+    def __init__(self, baseuri=None, baselang=None, encoding='utf-8'):

+        if _debug: sys.stderr.write('initializing FeedParser\n')

+        if not self._matchnamespaces:

+            for k, v in self.namespaces.items():

+                self._matchnamespaces[k.lower()] = v

+        self.feeddata = FeedParserDict() # feed-level data

+        self.encoding = encoding # character encoding

+        self.entries = [] # list of entry-level data

+        self.version = '' # feed type/version, see SUPPORTED_VERSIONS

+        self.namespacesInUse = {} # dictionary of namespaces defined by the feed

+

+        # the following are used internally to track state;

+        # this is really out of control and should be refactored

+        self.infeed = 0

+        self.inentry = 0

+        self.incontent = 0

+        self.intextinput = 0

+        self.inimage = 0

+        self.inauthor = 0

+        self.incontributor = 0

+        self.inpublisher = 0

+        self.insource = 0

+        self.sourcedata = FeedParserDict()

+        self.contentparams = FeedParserDict()

+        self._summaryKey = None

+        self.namespacemap = {}

+        self.elementstack = []

+        self.basestack = []

+        self.langstack = []

+        self.baseuri = baseuri or ''

+        self.lang = baselang or None

+        self.svgOK = 0

+        self.hasTitle = 0

+        if baselang:

+            self.feeddata['language'] = baselang.replace('_','-')

+

+    def unknown_starttag(self, tag, attrs):

+        if _debug: sys.stderr.write('start %s with %s\n' % (tag, attrs))

+        # normalize attrs

+        attrs = [(k.lower(), v) for k, v in attrs]

+        attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]

+

+        # track xml:base and xml:lang

+        attrsD = dict(attrs)

+        baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri

+        if type(baseuri) != type(u''):

+            try:

+                baseuri = unicode(baseuri, self.encoding)

+            except:

+                baseuri = unicode(baseuri, 'iso-8859-1')

+        self.baseuri = _urljoin(self.baseuri, baseuri)

+        lang = attrsD.get('xml:lang', attrsD.get('lang'))

+        if lang == '':

+            # xml:lang could be explicitly set to '', we need to capture that

+            lang = None

+        elif lang is None:

+            # if no xml:lang is specified, use parent lang

+            lang = self.lang

+        if lang:

+            if tag in ('feed', 'rss', 'rdf:RDF'):

+                self.feeddata['language'] = lang.replace('_','-')

+        self.lang = lang

+        self.basestack.append(self.baseuri)

+        self.langstack.append(lang)

+

+        # track namespaces

+        for prefix, uri in attrs:

+            if prefix.startswith('xmlns:'):

+                self.trackNamespace(prefix[6:], uri)

+            elif prefix == 'xmlns':

+                self.trackNamespace(None, uri)

+

+        # track inline content

+        if self.incontent and self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'):

+            if tag in ['xhtml:div', 'div']: return # typepad does this 10/2007

+            # element declared itself as escaped markup, but it isn't really

+            self.contentparams['type'] = 'application/xhtml+xml'

+        if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml':

+            if tag.find(':') <> -1:

+                prefix, tag = tag.split(':', 1)

+                namespace = self.namespacesInUse.get(prefix, '')

+                if tag=='math' and namespace=='http://www.w3.org/1998/Math/MathML':

+                    attrs.append(('xmlns',namespace))

+                if tag=='svg' and namespace=='http://www.w3.org/2000/svg':

+                    attrs.append(('xmlns',namespace))

+            if tag == 'svg': self.svgOK += 1

+            return self.handle_data('<%s%s>' % (tag, self.strattrs(attrs)), escape=0)

+

+        # match namespaces

+        if tag.find(':') <> -1:

+            prefix, suffix = tag.split(':', 1)

+        else:

+            prefix, suffix = '', tag

+        prefix = self.namespacemap.get(prefix, prefix)

+        if prefix:

+            prefix = prefix + '_'

+

+        # special hack for better tracking of empty textinput/image elements in illformed feeds

+        if (not prefix) and tag not in ('title', 'link', 'description', 'name'):

+            self.intextinput = 0

+        if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'):

+            self.inimage = 0

+

+        # call special handler (if defined) or default handler

+        methodname = '_start_' + prefix + suffix

+        try:

+            method = getattr(self, methodname)

+            return method(attrsD)

+        except AttributeError:

+            return self.push(prefix + suffix, 1)

+

+    def unknown_endtag(self, tag):

+        if _debug: sys.stderr.write('end %s\n' % tag)

+        # match namespaces

+        if tag.find(':') <> -1:

+            prefix, suffix = tag.split(':', 1)

+        else:

+            prefix, suffix = '', tag

+        prefix = self.namespacemap.get(prefix, prefix)

+        if prefix:

+            prefix = prefix + '_'

+        if suffix == 'svg' and self.svgOK: self.svgOK -= 1

+

+        # call special handler (if defined) or default handler

+        methodname = '_end_' + prefix + suffix

+        try:

+            if self.svgOK: raise AttributeError()

+            method = getattr(self, methodname)

+            method()

+        except AttributeError:

+            self.pop(prefix + suffix)

+

+        # track inline content

+        if self.incontent and self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'):

+            # element declared itself as escaped markup, but it isn't really

+            if tag in ['xhtml:div', 'div']: return # typepad does this 10/2007

+            self.contentparams['type'] = 'application/xhtml+xml'

+        if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml':

+            tag = tag.split(':')[-1]

+            self.handle_data('</%s>' % tag, escape=0)

+

+        # track xml:base and xml:lang going out of scope

+        if self.basestack:

+            self.basestack.pop()

+            if self.basestack and self.basestack[-1]:

+                self.baseuri = self.basestack[-1]

+        if self.langstack:

+            self.langstack.pop()

+            if self.langstack: # and (self.langstack[-1] is not None):

+                self.lang = self.langstack[-1]

+

+    def handle_charref(self, ref):

+        # called for each character reference, e.g. for ' ', ref will be '160'

+        if not self.elementstack: return

+        ref = ref.lower()

+        if ref in ('34', '38', '39', '60', '62', 'x22', 'x26', 'x27', 'x3c', 'x3e'):

+            text = '&#%s;' % ref

+        else:

+            if ref[0] == 'x':

+                c = int(ref[1:], 16)

+            else:

+                c = int(ref)

+            text = unichr(c).encode('utf-8')

+        self.elementstack[-1][2].append(text)

+

+    def handle_entityref(self, ref):

+        # called for each entity reference, e.g. for '©', ref will be 'copy'

+        if not self.elementstack: return

+        if _debug: sys.stderr.write('entering handle_entityref with %s\n' % ref)

+        if ref in ('lt', 'gt', 'quot', 'amp', 'apos'):

+            text = '&%s;' % ref

+        elif ref in self.entities.keys():

+            text = self.entities[ref]

+            if text.startswith('&#') and text.endswith(';'):

+                return self.handle_entityref(text)

+        else:

+            try: name2codepoint[ref]

+            except KeyError: text = '&%s;' % ref

+            else: text = unichr(name2codepoint[ref]).encode('utf-8')

+        self.elementstack[-1][2].append(text)

+

+    def handle_data(self, text, escape=1):

+        # called for each block of plain text, i.e. outside of any tag and

+        # not containing any character or entity references

+        if not self.elementstack: return

+        if escape and self.contentparams.get('type') == 'application/xhtml+xml':

+            text = _xmlescape(text)

+        self.elementstack[-1][2].append(text)

+

+    def handle_comment(self, text):

+        # called for each comment, e.g. <!-- insert message here -->

+        pass

+

+    def handle_pi(self, text):

+        # called for each processing instruction, e.g. <?instruction>

+        pass

+

+    def handle_decl(self, text):

+        pass

+

+    def parse_declaration(self, i):

+        # override internal declaration handler to handle CDATA blocks

+        if _debug: sys.stderr.write('entering parse_declaration\n')

+        if self.rawdata[i:i+9] == '<![CDATA[':

+            k = self.rawdata.find(']]>', i)

+            if k == -1: k = len(self.rawdata)

+            self.handle_data(_xmlescape(self.rawdata[i+9:k]), 0)

+            return k+3

+        else:

+            k = self.rawdata.find('>', i)

+            return k+1

+

+    def mapContentType(self, contentType):

+        contentType = contentType.lower()

+        if contentType == 'text':

+            contentType = 'text/plain'

+        elif contentType == 'html':

+            contentType = 'text/html'

+        elif contentType == 'xhtml':

+            contentType = 'application/xhtml+xml'

+        return contentType

+

+    def trackNamespace(self, prefix, uri):

+        loweruri = uri.lower()

+        if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/') and not self.version:

+            self.version = 'rss090'

+        if loweruri == 'http://purl.org/rss/1.0/' and not self.version:

+            self.version = 'rss10'

+        if loweruri == 'http://www.w3.org/2005/atom' and not self.version:

+            self.version = 'atom10'

+        if loweruri.find('backend.userland.com/rss') <> -1:

+            # match any backend.userland.com namespace

+            uri = 'http://backend.userland.com/rss'

+            loweruri = uri

+        if self._matchnamespaces.has_key(loweruri):

+            self.namespacemap[prefix] = self._matchnamespaces[loweruri]

+            self.namespacesInUse[self._matchnamespaces[loweruri]] = uri

+        else:

+            self.namespacesInUse[prefix or ''] = uri

+

+    def resolveURI(self, uri):

+        return _urljoin(self.baseuri or '', uri)

+

+    def decodeEntities(self, element, data):

+        return data

+

+    def strattrs(self, attrs):

+        return ''.join([' %s="%s"' % (t[0],_xmlescape(t[1],{'"':'"'})) for t in attrs])

+

+    def push(self, element, expectingText):

+        self.elementstack.append([element, expectingText, []])

+

+    def pop(self, element, stripWhitespace=1):

+        if not self.elementstack: return

+        if self.elementstack[-1][0] != element: return

+

+        element, expectingText, pieces = self.elementstack.pop()

+

+        if self.version == 'atom10' and self.contentparams.get('type','text') == 'application/xhtml+xml':

+            # remove enclosing child element, but only if it is a <div> and

+            # only if all the remaining content is nested underneath it.

+            # This means that the divs would be retained in the following:

+            #    <div>foo</div><div>bar</div>

+            while pieces and len(pieces)>1 and not pieces[-1].strip():

+                del pieces[-1]

+            while pieces and len(pieces)>1 and not pieces[0].strip():

+                del pieces[0]

+            if pieces and (pieces[0] == '<div>' or pieces[0].startswith('<div ')) and pieces[-1]=='</div>':

+                depth = 0

+                for piece in pieces[:-1]:

+                    if piece.startswith('</'):

+                        depth -= 1

+                        if depth == 0: break

+                    elif piece.startswith('<') and not piece.endswith('/>'):

+                        depth += 1

+                else:

+                    pieces = pieces[1:-1]

+

+        output = ''.join(pieces)

+        if stripWhitespace:

+            output = output.strip()

+        if not expectingText: return output

+

+        # decode base64 content

+        if base64 and self.contentparams.get('base64', 0):

+            try:

+                output = base64.decodestring(output)

+            except binascii.Error:

+                pass

+            except binascii.Incomplete:

+                pass

+

+        # resolve relative URIs

+        if (element in self.can_be_relative_uri) and output:

+            output = self.resolveURI(output)

+

+        # decode entities within embedded markup

+        if not self.contentparams.get('base64', 0):

+            output = self.decodeEntities(element, output)

+

+        if self.lookslikehtml(output):

+            self.contentparams['type']='text/html'

+

+        # remove temporary cruft from contentparams

+        try:

+            del self.contentparams['mode']

+        except KeyError:

+            pass

+        try:

+            del self.contentparams['base64']

+        except KeyError:

+            pass

+

+        is_htmlish = self.mapContentType(self.contentparams.get('type', 'text/html')) in self.html_types

+        # resolve relative URIs within embedded markup

+        if is_htmlish and RESOLVE_RELATIVE_URIS:

+            if element in self.can_contain_relative_uris:

+                output = _resolveRelativeURIs(output, self.baseuri, self.encoding, self.contentparams.get('type', 'text/html'))

+

+        # parse microformats

+        # (must do this before sanitizing because some microformats

+        # rely on elements that we sanitize)

+        if is_htmlish and element in ['content', 'description', 'summary']:

+            mfresults = _parseMicroformats(output, self.baseuri, self.encoding)

+            if mfresults:

+                for tag in mfresults.get('tags', []):

+                    self._addTag(tag['term'], tag['scheme'], tag['label'])

+                for enclosure in mfresults.get('enclosures', []):

+                    self._start_enclosure(enclosure)

+                for xfn in mfresults.get('xfn', []):

+                    self._addXFN(xfn['relationships'], xfn['href'], xfn['name'])

+                vcard = mfresults.get('vcard')

+                if vcard:

+                    self._getContext()['vcard'] = vcard

+

+        # sanitize embedded markup

+        if is_htmlish and SANITIZE_HTML:

+            if element in self.can_contain_dangerous_markup:

+                output = _sanitizeHTML(output, self.encoding, self.contentparams.get('type', 'text/html'))

+

+        if self.encoding and type(output) != type(u''):

+            try:

+                output = unicode(output, self.encoding)

+            except:

+                pass

+

+        # address common error where people take data that is already

+        # utf-8, presume that it is iso-8859-1, and re-encode it.

+        if self.encoding=='utf-8' and type(output) == type(u''):

+            try:

+                output = unicode(output.encode('iso-8859-1'), 'utf-8')

+            except:

+                pass

+

+        # map win-1252 extensions to the proper code points

+        if type(output) == type(u''):

+            output = u''.join([c in _cp1252.keys() and _cp1252[c] or c for c in output])

+

+        # categories/tags/keywords/whatever are handled in _end_category

+        if element == 'category':

+            return output

+

+        if element == 'title' and self.hasTitle:

+            return output

+

+        # store output in appropriate place(s)

+        if self.inentry and not self.insource:

+            if element == 'content':

+                self.entries[-1].setdefault(element, [])

+                contentparams = copy.deepcopy(self.contentparams)

+                contentparams['value'] = output

+                self.entries[-1][element].append(contentparams)

+            elif element == 'link':

+                self.entries[-1][element] = output

+                if output:

+                    self.entries[-1]['links'][-1]['href'] = output

+            else:

+                if element == 'description':

+                    element = 'summary'

+                self.entries[-1][element] = output

+                if self.incontent:

+                    contentparams = copy.deepcopy(self.contentparams)

+                    contentparams['value'] = output

+                    self.entries[-1][element + '_detail'] = contentparams

+        elif (self.infeed or self.insource):# and (not self.intextinput) and (not self.inimage):

+            context = self._getContext()

+            if element == 'description':

+                element = 'subtitle'

+            context[element] = output

+            if element == 'link':

+                context['links'][-1]['href'] = output

+            elif self.incontent:

+                contentparams = copy.deepcopy(self.contentparams)

+                contentparams['value'] = output

+                context[element + '_detail'] = contentparams

+        return output

+

+    def pushContent(self, tag, attrsD, defaultContentType, expectingText):

+        self.incontent += 1

+        if self.lang: self.lang=self.lang.replace('_','-')

+        self.contentparams = FeedParserDict({

+            'type': self.mapContentType(attrsD.get('type', defaultContentType)),

+            'language': self.lang,

+            'base': self.baseuri})

+        self.contentparams['base64'] = self._isBase64(attrsD, self.contentparams)

+        self.push(tag, expectingText)

+

+    def popContent(self, tag):

+        value = self.pop(tag)

+        self.incontent -= 1

+        self.contentparams.clear()

+        return value

+

+    # a number of elements in a number of RSS variants are nominally plain

+    # text, but this is routinely ignored.  This is an attempt to detect

+    # the most common cases.  As false positives often result in silent

+    # data loss, this function errs on the conservative side.

+    def lookslikehtml(self, str):

+        if self.version.startswith('atom'): return

+        if self.contentparams.get('type','text/html') != 'text/plain': return

+

+        # must have a close tag or a entity reference to qualify

+        if not (re.search(r'</(\w+)>',str) or re.search("&#?\w+;",str)): return

+

+        # all tags must be in a restricted subset of valid HTML tags

+        if filter(lambda t: t.lower() not in _HTMLSanitizer.acceptable_elements,

+            re.findall(r'</?(\w+)',str)): return

+

+        # all entities must have been defined as valid HTML entities

+        from htmlentitydefs import entitydefs

+        if filter(lambda e: e not in entitydefs.keys(),

+            re.findall(r'&(\w+);',str)): return

+

+        return 1

+

+    def _mapToStandardPrefix(self, name):

+        colonpos = name.find(':')

+        if colonpos <> -1:

+            prefix = name[:colonpos]

+            suffix = name[colonpos+1:]

+            prefix = self.namespacemap.get(prefix, prefix)

+            name = prefix + ':' + suffix

+        return name

+

+    def _getAttribute(self, attrsD, name):

+        return attrsD.get(self._mapToStandardPrefix(name))

+

+    def _isBase64(self, attrsD, contentparams):

+        if attrsD.get('mode', '') == 'base64':

+            return 1

+        if self.contentparams['type'].startswith('text/'):

+            return 0

+        if self.contentparams['type'].endswith('+xml'):

+            return 0

+        if self.contentparams['type'].endswith('/xml'):

+            return 0

+        return 1

+

+    def _itsAnHrefDamnIt(self, attrsD):

+        href = attrsD.get('url', attrsD.get('uri', attrsD.get('href', None)))

+        if href:

+            try:

+                del attrsD['url']

+            except KeyError:

+                pass

+            try:

+                del attrsD['uri']

+            except KeyError:

+                pass

+            attrsD['href'] = href

+        return attrsD

+

+    def _save(self, key, value):

+        context = self._getContext()

+        context.setdefault(key, value)

+

+    def _start_rss(self, attrsD):

+        versionmap = {'0.91': 'rss091u',

+                      '0.92': 'rss092',

+                      '0.93': 'rss093',

+                      '0.94': 'rss094'}

+        if not self.version:

+            attr_version = attrsD.get('version', '')

+            version = versionmap.get(attr_version)

+            if version:

+                self.version = version

+            elif attr_version.startswith('2.'):

+                self.version = 'rss20'

+            else:

+                self.version = 'rss'

+

+    def _start_dlhottitles(self, attrsD):

+        self.version = 'hotrss'

+

+    def _start_channel(self, attrsD):

+        self.infeed = 1

+        self._cdf_common(attrsD)

+    _start_feedinfo = _start_channel

+

+    def _cdf_common(self, attrsD):

+        if attrsD.has_key('lastmod'):

+            self._start_modified({})

+            self.elementstack[-1][-1] = attrsD['lastmod']

+            self._end_modified()

+        if attrsD.has_key('href'):

+            self._start_link({})

+            self.elementstack[-1][-1] = attrsD['href']

+            self._end_link()

+

+    def _start_feed(self, attrsD):

+        self.infeed = 1

+        versionmap = {'0.1': 'atom01',

+                      '0.2': 'atom02',

+                      '0.3': 'atom03'}

+        if not self.version:

+            attr_version = attrsD.get('version')

+            version = versionmap.get(attr_version)

+            if version:

+                self.version = version

+            else:

+                self.version = 'atom'

+

+    def _end_channel(self):

+        self.infeed = 0

+    _end_feed = _end_channel

+

+    def _start_image(self, attrsD):

+        context = self._getContext()

+        context.setdefault('image', FeedParserDict())

+        self.inimage = 1

+        self.hasTitle = 0

+        self.push('image', 0)

+

+    def _end_image(self):

+        self.pop('image')

+        self.inimage = 0

+

+    def _start_textinput(self, attrsD):

+        context = self._getContext()

+        context.setdefault('textinput', FeedParserDict())

+        self.intextinput = 1

+        self.hasTitle = 0

+        self.push('textinput', 0)

+    _start_textInput = _start_textinput

+

+    def _end_textinput(self):

+        self.pop('textinput')

+        self.intextinput = 0

+    _end_textInput = _end_textinput

+

+    def _start_author(self, attrsD):

+        self.inauthor = 1

+        self.push('author', 1)

+    _start_managingeditor = _start_author

+    _start_dc_author = _start_author

+    _start_dc_creator = _start_author

+    _start_itunes_author = _start_author

+

+    def _end_author(self):

+        self.pop('author')

+        self.inauthor = 0

+        self._sync_author_detail()

+    _end_managingeditor = _end_author

+    _end_dc_author = _end_author

+    _end_dc_creator = _end_author

+    _end_itunes_author = _end_author

+

+    def _start_itunes_owner(self, attrsD):

+        self.inpublisher = 1

+        self.push('publisher', 0)

+

+    def _end_itunes_owner(self):

+        self.pop('publisher')

+        self.inpublisher = 0

+        self._sync_author_detail('publisher')

+

+    def _start_contributor(self, attrsD):

+        self.incontributor = 1

+        context = self._getContext()

+        context.setdefault('contributors', [])

+        context['contributors'].append(FeedParserDict())

+        self.push('contributor', 0)

+

+    def _end_contributor(self):

+        self.pop('contributor')

+        self.incontributor = 0

+

+    def _start_dc_contributor(self, attrsD):

+        self.incontributor = 1

+        context = self._getContext()

+        context.setdefault('contributors', [])

+        context['contributors'].append(FeedParserDict())

+        self.push('name', 0)

+

+    def _end_dc_contributor(self):

+        self._end_name()

+        self.incontributor = 0

+

+    def _start_name(self, attrsD):

+        self.push('name', 0)

+    _start_itunes_name = _start_name

+

+    def _end_name(self):

+        value = self.pop('name')

+        if self.inpublisher:

+            self._save_author('name', value, 'publisher')

+        elif self.inauthor:

+            self._save_author('name', value)

+        elif self.incontributor:

+            self._save_contributor('name', value)

+        elif self.intextinput:

+            context = self._getContext()

+            context['name'] = value

+    _end_itunes_name = _end_name

+

+    def _start_width(self, attrsD):

+        self.push('width', 0)

+

+    def _end_width(self):

+        value = self.pop('width')

+        try:

+            value = int(value)

+        except:

+            value = 0

+        if self.inimage:

+            context = self._getContext()

+            context['width'] = value

+

+    def _start_height(self, attrsD):

+        self.push('height', 0)

+

+    def _end_height(self):

+        value = self.pop('height')

+        try:

+            value = int(value)

+        except:

+            value = 0

+        if self.inimage: