[puhutv] Improve extraction (closes #16269)

youtube_dl/extractor/puhutv.py

@@ -2,53 +2,54 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+)
 from ..utils import (
+    ExtractorError,
     int_or_none,
     float_or_none,
-    determine_ext,
+    parse_resolution,
     str_or_none,
-    url_or_none,
-    unified_strdate,
-    unified_timestamp,
     try_get,
-    url_basename,
+    unified_timestamp,
-    remove_end
+    url_or_none,
+    urljoin,
 )
 
 
 class PuhuTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[a-z0-9-]+)-izle'
+    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-izle'
     IE_NAME = 'puhutv'
-    _TESTS = [
+    _TESTS = [{
-        {
+        # film
-            # A Film
+        'url': 'https://puhutv.com/sut-kardesler-izle',
-            'url': 'https://puhutv.com/sut-kardesler-izle',
+        'md5': 'fbd8f2d8e7681f8bcd51b592475a6ae7',
-            'md5': 'a347470371d56e1585d1b2c8dab01c96',
+        'info_dict': {
-            'info_dict': {
+            'id': '5085',
-                'id': 'sut-kardesler',
+            'display_id': 'sut-kardesler',
-                'display_id': '5085',
+            'ext': 'mp4',
-                'ext': 'mp4',
+            'title': 'Süt Kardeşler',
-                'title': 'Süt Kardeşler',
+            'description': 'md5:405fd024df916ca16731114eb18e511a',
-                'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
-                'uploader': 'Arzu Film',
+            'duration': 4832.44,
-                'description': 'md5:405fd024df916ca16731114eb18e511a',
+            'creator': 'Arzu Film',
-                'uploader_id': '43',
+            'timestamp': 1469778212,
-                'upload_date': '20160729',
+            'upload_date': '20160729',
-                'timestamp': int,
+            'release_year': 1976,
-            },
+            'view_count': int,
-        },
+            'tags': ['Aile', 'Komedi', 'Klasikler'],
-        {
-            # An Episode and geo restricted
-            'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
-            'only_matching': True,
         },
-        {
+    }, {
-            # Has subtitle
+        # episode, geo restricted, bypassable with --geo-verification-proxy
-            'url': 'https://puhutv.com/dip-1-bolum-izle',
+        'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
-            'only_matching': True,
+        'only_matching': True,
-        }
+    }, {
-    ]
+        # 4k, with subtitles
+        'url': 'https://puhutv.com/dip-1-bolum-izle',
+        'only_matching': True,
+    }]
     _SUBTITLE_LANGS = {
         'English': 'en',
         'Deutsch': 'de',
@@ -56,47 +57,103 @@ class PuhuTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
+
         info = self._download_json(
-            'https://puhutv.com/api/slug/%s-izle' % video_id, video_id)['data']
+            urljoin(url, '/api/slug/%s-izle' % display_id),
+            display_id)['data']
 
-        display_id = compat_str(info['id'])
+        video_id = compat_str(info['id'])
-        title = info['title']['name']
+        title = info.get('name') or info['title']['name']
         if info.get('display_name'):
             title = '%s %s' % (title, info.get('display_name'))
 
-        description = try_get(info, lambda x: x['title']['description'], compat_str) or info.get('description')
+        try:
+            videos = self._download_json(
+                'https://puhutv.com/api/assets/%s/videos' % video_id,
+                display_id, 'Downloading video JSON',
+                headers=self.geo_verification_headers())
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                self.raise_geo_restricted()
+            raise
+
+        formats = []
+        for video in videos['data']['videos']:
+            media_url = url_or_none(video.get('url'))
+            if not media_url:
+                continue
+            playlist = video.get('is_playlist')
+            if video.get('stream_type') == 'hls' and playlist is True:
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue
+            quality = int_or_none(video.get('quality'))
+            f = {
+                'url': media_url,
+                'ext': 'mp4',
+                'height': quality
+            }
+            video_format = video.get('video_format')
+            if video_format == 'hls' and playlist is False:
+                format_id = 'hls'
+                f['protocol'] = 'm3u8_native'
+            elif video_format == 'mp4':
+                format_id = 'http'
+
+            else:
+                continue
+            if quality:
+                format_id += '-%sp' % quality
+            f['format_id'] = format_id
+            formats.append(f)
+        self._sort_formats(formats)
+
+        description = try_get(
+            info, lambda x: x['title']['description'],
+            compat_str) or info.get('description')
         timestamp = unified_timestamp(info.get('created_at'))
-        upload_date = unified_strdate(info.get('created_at'))
+        creator = try_get(
-        uploader = try_get(info, lambda x: x['title']['producer']['name'], compat_str)
+            info, lambda x: x['title']['producer']['name'], compat_str)
-        uploader_id = str_or_none(try_get(info, lambda x: x['title']['producer']['id']))
+
-        view_count = int_or_none(try_get(info, lambda x: x['content']['watch_count']))
+        duration = float_or_none(
-        duration = float_or_none(try_get(info, lambda x: x['content']['duration_in_ms']), scale=1000)
+            try_get(info, lambda x: x['content']['duration_in_ms'], int),
-        thumbnail = try_get(info, lambda x: x['content']['images']['wide']['main'], compat_str)
+            scale=1000)
-        release_year = int_or_none(try_get(info, lambda x: x['title']['released_at']))
+        view_count = try_get(info, lambda x: x['content']['watch_count'], int)
-        webpage_url = info.get('web_url')
+
+        images = try_get(
+            info, lambda x: x['content']['images']['wide'], dict) or {}
+        thumbnails = []
+        for image_id, image_url in images.items():
+            if not isinstance(image_url, compat_str):
+                continue
+            if not image_url.startswith(('http', '//')):
+                image_url = 'https://%s' % image_url
+            t = parse_resolution(image_id)
+            t.update({
+                'id': image_id,
+                'url': image_url
+            })
+            thumbnails.append(t)
+
+        release_year = try_get(info, lambda x: x['title']['released_at'], int)
 
         season_number = int_or_none(info.get('season_number'))
-        season_id = int_or_none(info.get('season_id'))
+        season_id = str_or_none(info.get('season_id'))
         episode_number = int_or_none(info.get('episode_number'))
 
         tags = []
-        for tag in try_get(info, lambda x: x['title']['genres'], list) or []:
+        for genre in try_get(info, lambda x: x['title']['genres'], list) or []:
-            if isinstance(tag.get('name'), compat_str):
+            if not isinstance(genre, dict):
-                tags.append(tag.get('name'))
-
-        thumbnails = []
-        thumbs_dict = try_get(info, lambda x: x['content']['images']['wide'], dict) or {}
-        for id, url in thumbs_dict.items():
-            if not url_or_none(url):
                 continue
-            thumbnails.append({
+            genre_name = genre.get('name')
-                'url': 'https://%s' % url,
+            if genre_name and isinstance(genre_name, compat_str):
-                'id': id
+                tags.append(genre_name)
-            })
 
         subtitles = {}
-        for subtitle in try_get(info, lambda x: x['content']['subtitles'], list) or []:
+        for subtitle in try_get(
+                info, lambda x: x['content']['subtitles'], list) or []:
             if not isinstance(subtitle, dict):
                 continue
             lang = subtitle.get('language')
@@ -107,30 +164,6 @@ class PuhuTVIE(InfoExtractor):
                 'url': sub_url
             }]
 
-        # Some of videos are geo restricted upon request copyright owner and returns 403
-        req_formats = self._download_json(
-            'https://puhutv.com/api/assets/%s/videos' % display_id,
-            video_id, 'Downloading video JSON')
-
-        formats = []
-        for format in req_formats['data']['videos']:
-            media_url = url_or_none(format.get('url'))
-            if not media_url:
-                continue
-            ext = format.get('video_format') or determine_ext(media_url)
-            quality = format.get('quality')
-            if format.get('stream_type') == 'hls' and format.get('is_playlist') is True:
-                m3u8_id = remove_end(url_basename(media_url), '.m3u8')
-                formats.append(self._m3u8_meta_format(media_url, ext, m3u8_id=m3u8_id))
-            elif ext == 'mp4' and format.get('is_playlist', False) is False:
-                formats.append({
-                    'url': media_url,
-                    'format_id': 'http-%s' % quality,
-                    'ext': ext,
-                    'height': quality
-                })
-        self._sort_formats(formats)
-
         return {
             'id': video_id,
             'display_id': display_id,
@@ -140,93 +173,75 @@ class PuhuTVIE(InfoExtractor):
             'season_number': season_number,
             'episode_number': episode_number,
             'release_year': release_year,
-            'upload_date': upload_date,
             'timestamp': timestamp,
-            'uploader': uploader,
+            'creator': creator,
-            'uploader_id': uploader_id,
             'view_count': view_count,
             'duration': duration,
             'tags': tags,
             'subtitles': subtitles,
-            'webpage_url': webpage_url,
-            'thumbnail': thumbnail,
             'thumbnails': thumbnails,
             'formats': formats
         }
 
 
 class PuhuTVSerieIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[a-z0-9-]+)-detay'
+    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-detay'
     IE_NAME = 'puhutv:serie'
-    _TESTS = [
+    _TESTS = [{
-        {
+        'url': 'https://puhutv.com/deniz-yildizi-detay',
-            'url': 'https://puhutv.com/deniz-yildizi-detay',
+        'info_dict': {
-            'info_dict': {
+            'title': 'Deniz Yıldızı',
-                'title': 'Deniz Yıldızı',
+            'id': 'deniz-yildizi',
-                'id': 'deniz-yildizi',
-                'uploader': 'Focus Film',
-                'uploader_id': 61,
-            },
-            'playlist_mincount': 234,
         },
-        {
+        'playlist_mincount': 205,
-            # a film detail page which is using same url with serie page
+    }, {
-            'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
+        # a film detail page which is using same url with serie page
-            'info_dict': {
+        'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
-                'title': 'Kaybedenler Kulübü',
+        'only_matching': True,
-                'id': 'kaybedenler-kulubu',
+    }]
-                'uploader': 'Tolga Örnek, Murat Dörtbudak, Neslihan Dörtbudak, Kemal Kaplanoğlu',
+
-                'uploader_id': 248,
+    def _extract_entries(self, seasons):
-            },
-            'playlist_mincount': 1,
-        },
-    ]
-
-    def _extract_entries(self, playlist_id, seasons):
         for season in seasons:
-            season_id = season['id']
+            season_id = season.get('id')
-            season_number = season.get('position')
+            if not season_id:
-            pagenum = 1
+                continue
+            page = 1
             has_more = True
             while has_more is True:
-                season_info = self._download_json(
+                season = self._download_json(
                     'https://galadriel.puhutv.com/seasons/%s' % season_id,
-                    playlist_id, 'Downloading season %s page %s' % (season_number, pagenum), query={
+                    season_id, 'Downloading page %s' % page, query={
-                        'page': pagenum,
+                        'page': page,
                         'per': 40,
                     })
-                for episode in season_info.get('episodes'):
+                episodes = season.get('episodes')
-                    video_id = episode['slugPath'].replace('-izle', '')
+                if isinstance(episodes, list):
-                    yield self.url_result(
+                    for ep in episodes:
-                        'https://puhutv.com/%s-izle' % video_id,
+                        slug_path = str_or_none(ep.get('slugPath'))
-                        PuhuTVIE.ie_key(), video_id)
+                        if not slug_path:
-                pagenum = pagenum + 1
+                            continue
-                has_more = season_info.get('hasMore', False)
+                        video_id = str_or_none(int_or_none(ep.get('id')))
+                        yield self.url_result(
+                            'https://puhutv.com/%s' % slug_path,
+                            ie=PuhuTVIE.ie_key(), video_id=video_id,
+                            video_title=ep.get('name') or ep.get('eventLabel'))
+                page += 1
+                has_more = season.get('hasMore')
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         info = self._download_json(
-            'https://puhutv.com/api/slug/%s-detay' % playlist_id, playlist_id)['data']
+            urljoin(url, '/api/slug/%s-detay' % playlist_id),
+            playlist_id)['data']
 
-        title = info.get('name')
-        uploader = try_get(info, lambda x: x['producer']['name'], compat_str)
-        uploader_id = try_get(info, lambda x: x['producer']['id'])
         seasons = info.get('seasons')
         if seasons:
-            entries = self._extract_entries(playlist_id, seasons)
+            return self.playlist_result(
-        else:
+                self._extract_entries(seasons), playlist_id, info.get('name'))
-            # For films, these are using same url with series
+
-            video_id = info['assets'][0]['slug']
+        # For films, these are using same url with series
-            return self.url_result(
+        video_id = info.get('slug') or info['assets'][0]['slug']
-                'https://puhutv.com/%s-izle' % video_id,
+        return self.url_result(
-                PuhuTVIE.ie_key(), video_id)
+            'https://puhutv.com/%s-izle' % video_id,
-
+            PuhuTVIE.ie_key(), video_id)
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': title,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'entries': entries,
-        }