concurrency examples

Author: ramalho

.gitignore

@@ -1,6 +1,7 @@
 concurrency/flags/img/*.gif
-concurrency/charfinder_index.pickle
+concurrency/charfinder/charfinder_index.pickle
 metaprog/oscon-schedule/data/schedule?_db
+concurrency/wikipedia/fixture/docroot/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

concurrency/charfinder/charfinder.html

@@ -7,20 +7,21 @@
     //(function() {
       var BASE_URL = 'http://127.0.0.1:8888/chars';
       var RESULTS_PER_REQUEST = 10;
-      var REQUEST_DELAY = 1000;  // in milliseconds
+      var REQUEST_DELAY = 100;  // in milliseconds
       var httpRequest = new XMLHttpRequest();
       httpRequest.onreadystatechange = processResponse;
 
       function requestMaker(start) {
         var makeRequest = function (event) {
           var query = document.getElementById('queryField').value;
-          var limit = RESULTS_PER_REQUEST;
-          httpRequest.open('GET', BASE_URL+'?query='+query+'&limit='+limit);
+          var stop = start + RESULTS_PER_REQUEST;
+          var params = '?query='+query+'&start='+start+'&stop='+stop;
+          httpRequest.open('GET', BASE_URL+params);
           httpRequest.send();
           document.getElementById('message').textContent = 'Query: ' + query;
           var table = document.getElementById('results');
           var tr;
-          while (tr = table.lastChild) table.removeChild(tr);
+          if (start == 0) while (tr = table.lastChild) table.removeChild(tr);
           return false; // don't submit form
         }
         return makeRequest;
@@ -104,7 +105,7 @@
         var table = document.getElementById('results');
         var tr;
         var characters = getSymbols(results.chars);
-        for (var i=results.start; i < results.stop; i++) {
+        for (var i=0; i < characters.length; i++) {
           ch = characters[i];
           if (ch == '\n') continue;
           if (ch == '\x00') break;
@@ -116,9 +117,10 @@
           tr.cells[1].appendChild(document.createTextNode(ch));
           tr.id = hexCode;
           table.appendChild(tr);
-          if (results.stop < results.total) {
-            setTimeout(requestMaker(results.stop)(), REQUEST_DELAY);
-          }
+        }
+        // setTimeout(getDescriptions, REQUEST_DELAY/2)
+        if (results.stop < results.total) {
+          setTimeout(requestMaker(results.stop), REQUEST_DELAY);
         }
       }
       window.onload = function() {

concurrency/charfinder/charfinder.py

@@ -98,6 +98,8 @@ def query_type(text):
 
 CharDescription = namedtuple('CharDescription', 'code_str char name')
 
+QueryResult = namedtuple('QueryResult', 'len items')
+
 class UnicodeNameIndex:
 
     def __init__(self, chars=None):
@@ -169,12 +171,14 @@ def find_chars(self, query, start=0, stop=None):
         if result_sets:
             result = result_sets[0].intersection(*result_sets[1:])
             result = sorted(result)  # must sort for consistency
-            for char in itertools.islice(result, start, stop):
-                yield char
+            result_iter = itertools.islice(result, start, stop)
+            return QueryResult(len(result),
+                               (char for char in result_iter))
+        return QueryResult(0, ())
 
     def find_codes(self, query, start=0, stop=None):
         return (ord(char) for char
-                in self.find_chars(query, start, stop))
+                in self.find_chars(query, start, stop).items)
 
     def describe(self, char):
         code_str = 'U+{:04X}'.format(ord(char))
@@ -185,6 +189,10 @@ def find_descriptions(self, query, start=0, stop=None):
         for char in self.find_chars(query, start, stop):
             yield self.describe(char)
 
+    def get_descriptions(self, chars):
+        for char in chars:
+            yield self.describe(char)
+
     def describe_str(self, char):
         return '{:7}\t{}\t{}'.format(*self.describe(char))
 

concurrency/charfinder/charfinder_index.pickle

@@ -75,30 +75,26 @@ def form(request):
 @asyncio.coroutine
 def get_chars(request):
     peername = request.transport.get_extra_info('peername')
-    query = request.GET.get('query', '')
-    limit = request.GET.get('query', 0)
     print('Request from: {}, GET data: {!r}'.format(peername, dict(request.GET)))
+    query = request.GET.get('query', '')
     if query:
         try:
             start = int(request.GET.get('start', 0))
             stop = int(request.GET.get('stop', sys.maxsize))
         except ValueError:
             raise web.HTTPBadRequest()
         stop = min(stop, start+RESULTS_PER_REQUEST)
-        chars = list(index.find_chars(query, start, stop))
+        num_results, chars = index.find_chars(query, start, stop)
     else:
-        chars = []
-        start = 0
-        stop = 0
-    num_results = len(chars)
+        raise web.HTTPBadRequest()
     text = ''.join(char if n % 64 else char+'\n'
             for n, char in enumerate(chars, 1))
     response_data = {'total': num_results, 'start': start, 'stop': stop}
     print('Response to query: {query!r}, start: {start}, stop: {stop}'.format(
           query=query, **response_data))
     response_data['chars'] = text
     json_obj = json.dumps(response_data)
-    print('Sending {} results'.format(num_results))
+    print('Sending {} characters'.format(len(text)))
     headers = {'Access-Control-Allow-Origin': '*'}
     return web.Response(content_type=TEXT_TYPE, headers=headers, text=json_obj)
 

concurrency/charfinder/http_charfinder2.py

@@ -43,7 +43,8 @@ def test_find_word_1_match(sample_index):
 
 
 def test_find_word_1_match_character_result(sample_index):
-    res = [name(char) for char in sample_index.find_chars('currency')]
+    res = [name(char) for char in
+           sample_index.find_chars('currency').items]
     assert res == ['EURO-CURRENCY SIGN']
 
 

concurrency/charfinder/test_charfinder.py

@@ -0,0 +1,64 @@
+import sys
+import argparse
+import os
+
+from daypicts import get_picture_url, validate_date, gen_dates
+from daypicts import NoPictureForDate
+from daypicts import POTD_PATH
+
+FIXTURE_DIR = 'fixture/'
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    date_help = 'YYYY-MM-DD or YYYY-MM or YYYY: year, month and day'
+    parser.add_argument('date', help=date_help)
+
+    args = parser.parse_args(argv)
+
+    try:
+        iso_parts = validate_date(args.date)
+    except ValueError as exc:
+        print('error:', exc.args[0])
+        parser.print_usage()
+        sys.exit(2)
+
+    dates = list(gen_dates(iso_parts))
+    if len(dates) == 1:
+        print('-> Date: ', dates[0])
+    else:
+        fmt = '-> {} days: {}...{}'
+        print(fmt.format(len(dates), dates[0], dates[-1]))
+
+    return dates, args
+
+
+def save_picture_urls(dates, save_path):
+    for date in dates:
+        try:
+            url = get_picture_url(date)
+        except NoPictureForDate as exc:
+            snippet = repr(exc)
+        else:
+            snippet = url.replace('http://', 'src="//') + '"'
+        print(date, end=' ')
+        print(snippet)
+        with open(os.path.join(save_path, date), 'w') as fp:
+            fp.write(snippet)
+
+
+def main(argv):
+    """Build test fixture from Wikipedia "POTD" data"""
+
+    save_path = os.path.join(FIXTURE_DIR,POTD_PATH)
+    try:
+        os.makedirs(save_path)
+    except FileExistsError:
+        pass
+
+    dates, args = parse_args(argv)
+
+    save_picture_urls(dates, save_path)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

concurrency/wikipedia/build_fixture.py

@@ -25,7 +25,8 @@
 import requests
 
 SAVE_DIR = 'pictures/'
-POTD_BASE_URL = 'http://en.wikipedia.org/wiki/Template:POTD/'
+POTD_PATH = 'Template:POTD/'
+POTD_BASE_URL = 'http://en.wikipedia.org/wiki/' + POTD_PATH
 POTD_IMAGE_RE = re.compile(r'src="(//upload\..*?)"')
 PODT_EARLIEST_TEMPLATE = '2007-01-01'
 
@@ -84,7 +85,7 @@ def validate_date(text):
     test_parts = parts[:]
     while len(test_parts) < 3:
         test_parts.append(1)
-    date = datetime.datetime(*(int(part) for part in test_parts))
+    date = datetime.date(*(int(part) for part in test_parts))
     iso_date = date.strftime(ISO_DATE_FMT)
     iso_date = iso_date[:1+len(parts)*3]
     if iso_date < PODT_EARLIEST_TEMPLATE:
@@ -95,7 +96,7 @@ def validate_date(text):
 def gen_month_dates(iso_month):
     first = datetime.datetime.strptime(iso_month+'-01', ISO_DATE_FMT)
     one_day = datetime.timedelta(days=1)
-    date = first
+    date = first.date()
     while date.month == first.month:
         yield date.strftime(ISO_DATE_FMT)
         date += one_day
@@ -115,6 +116,26 @@ def gen_dates(iso_parts):
         yield iso_parts
 
 
+def get_picture_urls(dates, verbose=False, save_fixture=False):
+    urls = []
+    count = 0
+    for date in dates:
+        try:
+            url = get_picture_url(date)
+        except NoPictureForDate as exc:
+            if verbose:
+                print('*** {!r} ***'.format(exc))
+            continue
+        count += 1
+        if verbose:
+            print(format(count, '3d'), end=' ')
+            print(url.split('/')[-1])
+        else:
+            print(url)
+        urls.append(url)
+    return urls
+
+
 def parse_args(argv):
     parser = argparse.ArgumentParser(description=main.__doc__)
     date_help = 'YYYY-MM-DD or YYYY-MM or YYYY: year, month and day'
@@ -123,6 +144,8 @@ def parse_args(argv):
                         help='maximum number of items to fetch')
     parser.add_argument('-u', '--url_only', action='store_true',
                         help='get picture URLS only')
+    parser.add_argument('-f', '--fixture_save', action='store_true',
+                        help='save data for local test fixture')
     parser.add_argument('-v', '--verbose', action='store_true',
                         help='display progress information')
     args = parser.parse_args(argv)
@@ -145,34 +168,14 @@ def parse_args(argv):
     return dates, args
 
 
-def get_picture_urls(dates, verbose=False):
-    urls = []
-    count = 0
-    for date in dates:
-        try:
-            url = get_picture_url(date)
-        except NoPictureForDate as exc:
-            if verbose:
-                print('*** {!r} ***'.format(exc))
-            continue
-        count += 1
-        if verbose:
-            print(format(count, '3d'), end=' ')
-            print(url.split('/')[-1])
-        else:
-            print(url)
-        urls.append(url)
-    return urls
-
-
 def main(argv, get_picture_urls):
     """Get Wikipedia "Picture of The Day" for date, month or year"""
 
     dates, args = parse_args(argv)
 
     t0 = time.time()
 
-    urls = get_picture_urls(dates, args.verbose)
+    urls = get_picture_urls(dates, args.verbose, args.fixture_save)
 
     elapsed = time.time() - t0
     if args.verbose:

concurrency/wikipedia/daypicts.py

@@ -6,19 +6,19 @@
 import asyncio
 import aiohttp
 
-from daypicts import main
-from daypicts import NoPictureForDate
-from daypicts import POTD_BASE_URL
-from daypicts import POTD_IMAGE_RE
+from daypicts import main, NoPictureForDate
+from daypicts import POTD_BASE_URL, POTD_IMAGE_RE
 
 GLOBAL_TIMEOUT = 300  # seconds
+MAX_CONCURRENT_REQUESTS = 30
 
 
 @asyncio.coroutine
-def get_picture_url(iso_date):
+def get_picture_url(iso_date, semaphore):
     page_url = POTD_BASE_URL+iso_date
-    response = yield from aiohttp.request('GET', page_url)
-    text = yield from response.text()
+    with (yield from semaphore):
+        response = yield from aiohttp.request('GET', page_url)
+        text = yield from response.text()
     pict_url = POTD_IMAGE_RE.search(text)
     if pict_url is None:
         raise NoPictureForDate(iso_date)
@@ -27,7 +27,8 @@ def get_picture_url(iso_date):
 
 @asyncio.coroutine
 def get_picture_urls(dates, verbose=False):
-    tasks = [get_picture_url(date) for date in dates]
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+    tasks = [get_picture_url(date, semaphore) for date in dates]
     urls = []
     count = 0
     # get results as jobs are done

concurrency/wikipedia/daypicts_asyncio.py

@@ -7,13 +7,12 @@
 
 from daypicts import main, get_picture_url, NoPictureForDate
 
-MAX_NUM_THREADS = 400
 GLOBAL_TIMEOUT = 300  # seconds
+MAX_CONCURRENT_REQUESTS = 30
 
 
 def get_picture_urls(dates, verbose=False):
-    num_threads = min(len(dates), MAX_NUM_THREADS)
-    pool = futures.ThreadPoolExecutor(num_threads)
+    pool = futures.ThreadPoolExecutor(MAX_CONCURRENT_REQUESTS)
 
     pending = {}
     for date in dates: