sitesize/sitesize.py (de976a0) - testpython.git

sitesize.py

#!/usr/bin/env python3
# From https://pybay.com/site_media/slides/raymond2017-keynote/process.html
import urllib.request
import urllib.error
from multiprocessing.pool import ThreadPool

sites = [
#    'https://www.yahoo.com/',
#    'http://www.cnn.com',
#    'http://www.python.org',
#    'http://www.jython.org',
    'http://www.pypy.org',
    'http://www.perl.org',
#    'http://www.cisco.com',
#    'http://www.facebook.com',
#    'http://www.twitter.com',
#    'http://www.macrumors.com/',
#    'http://arstechnica.com/',
#    'http://www.reuters.com/',
#    'http://abcnews.go.com/',
#    'http://www.cnbc.com/',
]

def sitesize(url: str):
    ''' Determine the size of a website '''
    try:
        with urllib.request.urlopen(url) as f:
            page = f.read()
        return url, len(page)
    except urllib.error.HTTPError as e:
        return url, str(e)

def run():
    pool = ThreadPool(2)
    for result in pool.imap_unordered(sitesize, sites):
        print(result)

if __name__ == '__main__':
    run()

testpython.git

Make multithreaded with ThreadPool