diff --git a/test/test_utils.py b/test/test_utils.py
index be1069105..ff2e9885b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -11,13 +11,16 @@ import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
#from youtube_dl.utils import htmlentity_transform
-from youtube_dl.utils import timeconvert
-from youtube_dl.utils import sanitize_filename
-from youtube_dl.utils import unescapeHTML
-from youtube_dl.utils import orderedSet
-from youtube_dl.utils import DateRange
-from youtube_dl.utils import unified_strdate
-from youtube_dl.utils import find_xpath_attr
+from youtube_dl.utils import (
+ timeconvert,
+ sanitize_filename,
+ unescapeHTML,
+ orderedSet,
+ DateRange,
+ unified_strdate,
+ find_xpath_attr,
+ get_meta_content,
+)
if sys.version_info < (3, 0):
_compat_str = lambda b: b.decode('unicode-escape')
@@ -127,5 +130,16 @@ class TestUtil(unittest.TestCase):
self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
+ def test_meta_parser(self):
+ testhtml = u'''
+
+
+
+
+ '''
+ get_meta = lambda name: get_meta_content(name, testhtml)
+ self.assertEqual(get_meta('description'), u'foo & bar')
+ self.assertEqual(get_meta('author'), 'Plato')
+
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index f69b27d44..74c82587f 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -4,7 +4,7 @@ import re
from .common import InfoExtractor
from ..utils import (
compat_urlparse,
- compat_html_parser,
+ get_meta_content,
)
@@ -49,40 +49,6 @@ class UstreamIE(InfoExtractor):
}
return info
-# More robust than regular expressions
-
-class ChannelParser(compat_html_parser.HTMLParser):
- """
-
- """
- channel_id = None
-
- def handle_starttag(self, tag, attrs):
- if tag != 'meta':
- return
- values = dict(attrs)
- if values.get('name') != 'ustream:channel_id':
- return
- value = values.get('content', '')
- if value.isdigit():
- self.channel_id = value
-
-class SocialstreamParser(compat_html_parser.HTMLParser):
- """
-
- """
- def __init__(self):
- compat_html_parser.HTMLParser.__init__(self)
- self.content_ids = []
-
- def handle_starttag(self, tag, attrs):
- if tag != 'li':
- return
- for (attr, value) in attrs:
- if attr == 'data-content-id' and value.isdigit():
- self.content_ids.append(value)
-
class UstreamChannelIE(InfoExtractor):
_VALID_URL = r'https?://www\.ustream\.tv/channel/(?P.+)'
IE_NAME = u'ustream:channel'
@@ -90,21 +56,16 @@ class UstreamChannelIE(InfoExtractor):
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
slug = m.group('slug')
+ webpage = self._download_webpage(url, slug)
+ channel_id = get_meta_content('ustream:channel_id', webpage)
- p = ChannelParser()
- p.feed(self._download_webpage(url, slug))
- p.close()
- channel_id = p.channel_id
-
- p = SocialstreamParser()
BASE = 'http://www.ustream.tv'
next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
+ video_ids = []
while next_url:
reply = json.loads(self._download_webpage(compat_urlparse.urljoin(BASE, next_url), channel_id))
- p.feed(reply['data'])
+ video_ids.extend(re.findall(r'data-content-id="(\d.*)"', reply['data']))
next_url = reply['nextUrl']
- p.close()
- video_ids = p.content_ids
urls = ['http://www.ustream.tv/recorded/' + vid for vid in video_ids]
url_entries = [self.url_result(eurl, 'Ustream') for eurl in urls]
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 201802cee..768c6207d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -249,7 +249,17 @@ def htmlentity_transform(matchobj):
return (u'&%s;' % entity)
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class AttrParser(compat_html_parser.HTMLParser):
+class BaseHTMLParser(compat_html_parser.HTMLParser):
+ def __init(self):
+ compat_html_parser.HTMLParser.__init__(self)
+ self.html = None
+
+ def loads(self, html):
+ self.html = html
+ self.feed(html)
+ self.close()
+
+class AttrParser(BaseHTMLParser):
"""Modified HTMLParser that isolates a tag with the specified attribute"""
def __init__(self, attribute, value):
self.attribute = attribute
@@ -257,10 +267,9 @@ class AttrParser(compat_html_parser.HTMLParser):
self.result = None
self.started = False
self.depth = {}
- self.html = None
self.watch_startpos = False
self.error_count = 0
- compat_html_parser.HTMLParser.__init__(self)
+ BaseHTMLParser.__init__(self)
def error(self, message):
if self.error_count > 10 or self.started:
@@ -269,11 +278,6 @@ class AttrParser(compat_html_parser.HTMLParser):
self.error_count += 1
self.goahead(1)
- def loads(self, html):
- self.html = html
- self.feed(html)
- self.close()
-
def handle_starttag(self, tag, attrs):
attrs = dict(attrs)
if self.started:
@@ -334,6 +338,38 @@ def get_element_by_attribute(attribute, value, html):
pass
return parser.get_result()
+class MetaParser(BaseHTMLParser):
+ """
+ Modified HTMLParser that isolates a meta tag with the specified name
+ attribute.
+ """
+ def __init__(self, name):
+ BaseHTMLParser.__init__(self)
+ self.name = name
+ self.content = None
+ self.result = None
+
+ def handle_starttag(self, tag, attrs):
+ if tag != 'meta':
+ return
+ attrs = dict(attrs)
+ if attrs.get('name') == self.name:
+ self.result = attrs.get('content')
+
+ def get_result(self):
+ return self.result
+
+def get_meta_content(name, html):
+ """
+ Return the content attribute from the meta tag with the given name attribute.
+ """
+ parser = MetaParser(name)
+ try:
+ parser.loads(html)
+ except compat_html_parser.HTMLParseError:
+ pass
+ return parser.get_result()
+
def clean_html(html):
"""Clean an HTML snippet into a readable string"""