Changeset 1760

Timestamp:

08/09/08 13:51:33 (4 months ago)

Author:

duncan

svm:headrev:

cc3e1ea1-1e01-0410-8d68-8b121e83a9d5:10987

Message:

Fixed vpodcast:
- this crashed quite a lot
- uses youtube downloader, which needed to be renamed to youtube_dl.py
- made the buffering a lot faster, starts playing when there is something downloaded
- corrected the file extensions
Added .mp4 to freevo_config and local_conf.py.example
- the nocache option is good to mplayer when playing and downloading at the same time

Location:

freevo

Files:

• freevo_config.py (modified) (3 diffs)
• local_conf.py.example (modified) (2 diffs)
• src/video/plugins/vpodcast.py (modified) (11 diffs)

freevo/freevo_config.py

@@ -109 +109 @@
 # is different, there will be only a warning
 
-LOCAL_CONF_VERSION = 5.25
+LOCAL_CONF_VERSION = 5.26
 
 # Description of changes in each new version
@@ -379 +379 @@
      Added SYS_USE_MOUSE option for enabling mouse support if needed. False by default
      """),
+    (5.26,
+     """
+     """),
 ]
 
@@ -1592 +1595 @@
     'avi'    : '-cache 5000 -idx',
     'flv'    : '-nocache -forceidx',
+    'mp4'    : '-nocache -forceidx',
     'rm'     : '-cache 5000 -forceidx',
     'rmvb'   : '-cache 5000 -forceidx',

freevo/local_conf.py.example

@@ -51 +51 @@
 # -----------------------------------------------------------------------
 
-CONFIG_VERSION = 5.23
+CONFIG_VERSION = 5.26
 
 # ======================================================================
@@ -1127 +1127 @@
 # MPLAYER_ARGS['avi']     = '-cache 5000 -idx'
 # MPLAYER_ARGS['flv']     = '-nocache -forceidx'
+# MPLAYER_ARGS['mp4']     = '-nocache -forceidx'
 # MPLAYER_ARGS['rm']      = '-cache 5000 -forceidx'
 # MPLAYER_ARGS['rmvb']    = '-cache 5000 -forceidx'

freevo/src/video/plugins/vpodcast.py

@@ -38 +38 @@
 from gui import AlertBox, PopupBox, GUIObject
 from event import *
-#import youtube-dl
+import util.youtube_dl as youtube
+from util.benchmark import benchmark
+benchmarking = config.DEBUG_BENCHMARKING
+from pprint import pformat, pprint
 
 MAX_AGE = 3600 * 10
@@ -44 +47 @@
 _player_ = None
 
+
+
+class PodcastException: pass
 
 
@@ -66 +72 @@
     | ]
     |
-    | VPODCAST_DIR = '/home/user_name/VPODCAST'
-    """
+    | VPODCAST_DIR = '/path/to/vpodcasts'
+    """
+    @benchmark(benchmarking)
     def __init__(self):
         """ Initialise the Video postcast plug-in interface """
+        if config.VPODCAST_DIR is None:
+            self.reason = 'VPODCAST_DIR not set'
+            return
         plugin.MainMenuPlugin.__init__(self)
         self.plugin_name = 'vpodcast'
@@ -75 +85 @@
 
 
+    @benchmark(benchmarking)
     def config(self):
-        """ freevo plugins -i audio.vpodcast returns the info """
+        """ freevo plugins -i video.vpodcast returns the info """
         return [
             ('VPODCAST_LOCATIONS', None, 'List of podcast locations'),
-            ('VPODCAST_DIR', None, 'Directory for downloaded podcasts')
+            ('VPODCAST_DIR', os.path.join(config.FREEVO_CACHEDIR, 'vposcasts'), 'Directory for downloaded podcasts'),
+            ('YOUTUBE_USERNAME', None, 'YouTube user name (optional)'),
+            ('YOUTUBE_PASSWORD', None, 'YouTube password (optional)'),
+            ('YOUTUBE_FORMAT', '18', 'YouTube format 18=high 17=mobile (optional)'),
+            ('VPODCAST_BUFFERING_TIME', 20, 'Length of time to wait while buffering the poscast'),
+            ('VPODCAST_BUFFERING_SIZE', 20*1024, 'size of downloaded file to start playing'),
         ]
 
 
+    @benchmark(benchmarking)
     def items(self, parent):
         return [ VPodcastMainMenuItem(parent) ]
 
 
+    @benchmark(benchmarking)
     def check_dir(self):
         """ Check that the VPODCAST_DIR directories exist, if not create them """
@@ -94 +112 @@
 
         for pcdir in config.VPODCAST_LOCATIONS:
-            pc_dir = config.VPODCAST_DIR + '/' + pcdir[0]
-            if not os.path.isdir(pc_dir):
-                os.makedirs(pc_dir)
-
-
-class VPVideoItem(VideoItem):
-    """
-    Video podcast video item
-    """
-    def __init__(self, name, url, parent):
-        """ Initialise the VPVideoItem class """
-        self.vp_url = url
-        url = name
-        VideoItem.__init__(self, name, parent)
-
-
-    def play(self, arg=None, menuw=None):
-        """ Play this Podcast"""
-
-        # play the item.
-        isYT = self.vp_url.find('youtube.com')  #YouTube podcast
-        isMC = self.vp_url.find('metacafe.com') #Metacafe podcast
-
-        if isYT != -1:
-            self.download_url = self.youtube(self.vp_url)
-
-        elif isMC != -1:
-            self.download_url = self.metacafe(self.vp_url)
-
-        else:
-            self.download_url = self.vp_url
-
-        if not os.path.exists(self.filename):
-            background = BGDownload(self.download_url, self.filename)
-            background.start()
-            popup = PopupBox(text=_('Buffering podcast...'))
-            popup.show()
-            time.sleep(20) # 20s. buffering time
-            popup.destroy()
-
-        # call the play funuction of VideoItem
-        VideoItem.play(self, menuw=menuw, arg=arg)
-
-
-    def youtube(self, url):
-        const_video_url_str = 'http://www.youtube.com/watch?v=%s'
-        const_video_url_re = re.compile(
-        r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?(?:.+&)?v=))?([0-9A-Za-z_-]+)(?(1)[&/].*)?$')
-        const_url_t_param_re = re.compile(r', "t": "([^"]+)"')
-        const_video_url_real_str = 'http://www.youtube.com/get_video?video_id=%s&t=%s'
-        const_video_title_re = re.compile(r'<title>YouTube - ([^<]*)</title>', re.M | re.I)
-        try:
-            video_url_mo = const_video_url_re.match(url)
-            video_url_id = video_url_mo.group(2)
-            video_url = const_video_url_str % video_url_id
-            # Retrieve video webpage
-            video_webpage = urllib.urlopen(video_url).read()
-            match = const_url_t_param_re.search(video_webpage)
-            if match is None:
-                _debug_('No matches found for youtube', DWARNING)
-            video_url_t_param = match.group(1)
-            # Retrieve real video URL
-            video_url_real = const_video_url_real_str % (video_url_id, video_url_t_param)
-            return video_url_real
-        except Exception, why:
-            _debug_('Cannot read youtube URL: %s' (why,), DWARNING)
-
-
-    def metacafe(self, url):
-        video_url =  url
-        const_video_url_re = re.compile(r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+/)?.*')
-        const_normalized_url_str = 'http://www.metacafe.com/watch/%s/'
-        const_age_post_data = r'allowAdultContent=1&submit=Continue+-+I%27m+over+18'
-        const_video_mediaurl_re = re.compile(r'&mediaURL=([^&]+)&', re.M)
-
-        try:
-            # Verify video URL format and extract URL data to normalize URL
-            video_url_mo = const_video_url_re.match(video_url)
-            if video_url_mo is None:
-                _debug_('No matches found for metacafe', DWARNING)
-            video_url_id = video_url_mo.group(1)
-            video_url_title = (video_url_mo.group(2) is not None) and video_url_mo.group(2)[:-1] or None
-            video_url = const_normalized_url_str % video_url_id
-
-            # Retrieve video webpage
-            video_webpage = urllib.urlopen(video_url).read()
-            # Retrieve real video URL
-            video_url_real = self.extract_step('Extracting real video URL', 'unable to extract real video URL', \
-                const_video_mediaurl_re, video_webpage)
-            return video_url_real
-        except Exception, why:
-            _debug_('Cannot read metacafe URL: %s' (why,), DWARNING)
-
-
-    def extract_step(self, step_title, step_error, regexp, data):
-        try:
-            match = regexp.search(data)
-            if match is None:
-                error_advice_exit(step_error)
-
-            extracted_data = match.group(1)
-
-            return extracted_data
-
-        except KeyboardInterrupt:
-            sys.exit('\n')
+            podcastdir = os.path.join(config.VPODCAST_DIR, pcdir[0].strip().replace(' ', '_').lower())
+            if not os.path.isdir(podcastdir):
+                os.makedirs(podcastdir)
 
 
@@ -208 +123 @@
     of commands in a submenu.
     """
+    @benchmark(benchmarking)
     def __init__(self, parent):
         MenuItem.__init__(self, parent, arg='audio', skin_type='radio')
@@ -213 +129 @@
 
 
+    @benchmark(benchmarking)
     def actions(self):
         """ return a list of actions for this item """
@@ -218 +135 @@
 
 
+    @benchmark(benchmarking)
     def create_podcast_submenu(self, arg=None, menuw=None, image=None):
         """ create the sub-menu for the podcast """
+        dir, url = arg
+        podcastdir = dir.strip().replace(' ', '_').lower()
+
         popup = PopupBox(text=_('Fetching podcast...'))
         popup.show()
-        url = arg[1]
-        p = podcast()
-        p.open_rss(url)
-        p.rss_title()
-        p.rss_count()
-
-        podcast_items = []
-        for pc_location in range(p.rss.count):
-            p.rss_item(pc_location)
-            if p.image != None:
-                image = config.VPODCAST_DIR + '/' + arg[0] + '/' + p.title + '.jpg'
-                self.download(p.image, image)
+        try:
+            p = Podcast(url)
+            feed = p.feed()
+            rss_title = p.rss_title()
+            rss_description = p.rss_description()
+            rss_imageurl = p.rss_image()
+            if rss_imageurl:
+                filename = rss_title.strip().replace(' ', '_').lower()
+                image = os.path.join(config.VPODCAST_DIR, podcastdir, filename+'.jpg')
+                self.download(rss_imageurl, image)
             else:
                 image = None
-            url = p.link
-            name = p.title
-            if url != 'ERROR':
-                isYT = url.find('youtube.com')
-                isMC = url.find('metacafe.com')
-                if isYT == -1 and isMC == -1:
-                    file_ext = '.avi'
-                else:
-                    file_ext = '.flv'
-
-                filename  = config.VPODCAST_DIR + '/' + arg[0] + '/' + name + file_ext
-                podcast_items += [menu.MenuItem(_(p.title), \
-                    action=VPVideoItem(filename, url, self), arg=None, image=image)]
-
-        popup.destroy()
-        if (len(podcast_items) == 0):
-            podcast_items += [menu.MenuItem(_('No Podcast locations found'),
-                                             menwu.goto_prev_page, 0)]
+
+            podcast_items = []
+            for item in feed.entries:
+                #pprint(item) #DJW
+                try:
+                    item_title = p.rss_item_title(item)
+                    item_image = p.rss_item_image(item)
+                    item_link = p.rss_item_link(item)
+                    isYT = item_link.find('youtube.com')
+                    isMC = item_link.find('metacafe.com')
+                    name = item_title.strip().replace(' ', '_').lower()
+                    file_ext = isYT >= 0 and config.YOUTUBE_FORMAT == '18' and '.mp4' \
+                            or isYT >= 0 and config.YOUTUBE_FORMAT == '17' and '.3gp' \
+                            or isMC >= 0 and '.flv' \
+                            or '.avi'
+                    filename = os.path.join(config.VPODCAST_DIR, podcastdir, name+file_ext)
+                    podcast_items += [ menu.MenuItem(item_title, action=VPVideoItem(filename, item_link, self),
+                        arg=None, image=image) ]
+                except PodcastException:
+                    pass
+            if not podcast_items:
+                podcast_items += [menu.MenuItem(_('No Podcast locations found'), menuw.goto_prev_page, 0)]
+        finally:
+            popup.destroy()
+
         podcast_sub_menu = menu.Menu(_('Video Podcasts'), podcast_items)
-        rc.app(None)
         menuw.pushmenu(podcast_sub_menu)
         menuw.refresh()
-
-
+        rc.app(None)
+
+
+    @benchmark(benchmarking)
     def create_podcast_menu(self, arg=None, menuw=None):
         """ Create the main menu item for the video podcasts """
-        popup = PopupBox(text=_('Fetching podcasts...'))
+        popup = PopupBox(text=_('Fetching podcast list...'))
         popup.show()
         podcast_menu_items = []
 
-        for location in config.VPODCAST_LOCATIONS:
-            url = location[1]
-            image_path = config.VPODCAST_DIR + '/' + location[0] + '/' + 'cover.jpg'
-            if self.check_logo(image_path):
-                p = podcast()
-                p.open_rss(url)
-                p.rss_title()
-                name = p.rss_title
-                try:
-                    image_url = p.rss_image
-                    self.download(image_url, image_path)
-                except:
-                    _debug_('No image in RSS', DINFO)
-
-            if (len(config.VPODCAST_DIR) == 0):
-                podcast_items += [menu.MenuItem(_('Set VPODCAST_DIR in local_conf.py'), menwu.goto_prev_page, 0)]
-
-            podcast_menu_items += [menu.MenuItem(_(location[0]), action=self.create_podcast_submenu, \
-                arg=location, image=image_path)]
+        for dir, url in config.VPODCAST_LOCATIONS:
+            podcastdir = dir.strip().replace(' ', '_').lower()
+            image_path = os.path.join(config.VPODCAST_DIR, podcastdir, 'cover.jpg')
+            #XXX here we are only downloading the images, but this slows down the plug-in lots
+            #if self.check_logo(image_path):
+            #    p = Podcast()
+            #    p.open_rss(url)
+            #    p.rss_title()
+            #    #XXX name = p.rss_title
+            #    if p.rss_imageurl:
+            #        try:
+            #            image_url = p.rss_imageurl
+            #            self.download(image_url, image_path)
+            #        except Exception, why:
+            #            _debug_(why, DWARNING)
+
+            podcast_menu_items += [ menu.MenuItem(dir, action=self.create_podcast_submenu,
+                arg=(dir, url), image=image_path) ]
 
         popup.destroy()
@@ -293 +219 @@
 
 
+    @benchmark(benchmarking)
     def download(self, url, savefile):
         """ Download the url and save it """
@@ -301 +228 @@
 
 
+    @benchmark(benchmarking)
     def check_logo(self, logo_file):
-        """ Check if the logo has changed """
-        if os.path.exists(logo_file) == 0 or (abs(time.time() - os.path.getmtime(logo_file)) > MAX_AGE):
-            return True
-        else:
-            return False
-
-
-
-class podcast:
-    """
-    """
-    def __init__(self):
-        pass
-
-
-    def open_rss(self, url):
-        self.rss = util.feedparser.parse(url)
-        self.encoding = self.rss.encoding
-
-
+        """
+        Check if the logo has changed
+        @returns: True if the logo does not exist or is too old
+        """
+        return not os.path.exists(logo_file) or (time.time() - os.path.getmtime(logo_file)) > MAX_AGE
+
+
+
+class Podcast:
+    """
+    This bit of code really needs fixing, it's really bad
+    """
+    @benchmark(benchmarking)
+    def __init__(self, url):
+        self.url = url
+        self.encoding = 'latin-1'
+
+
+    @benchmark(benchmarking)
+    def feed(self):
+        try:
+            self.rss = util.feedparser.parse(self.url)
+            self.encoding = self.rss.encoding
+            return self.rss
+        except Exception, why:
+            _debug_('Cannot parse feed "%s": %s' % (self.url, why), DWARNING)
+            return None
+
+
+    @benchmark(benchmarking)
     def rss_title(self):
+        if self.rss.feed.has_key('title'):
+            return self.rss.feed.title.encode(self.encoding)
+        return None
+
+
+    @benchmark(benchmarking)
+    def rss_description(self):
+        if self.rss.feed.has_key('description'):
+            return self.rss.feed.description.encode(self.encoding)
+        return None
+
+
+    @benchmark(benchmarking)
+    def rss_image(self):
+        if self.rss.feed.has_key('image') and self.rss.feed.image.has_key('url'):
+            return self.rss.feed.image.url
+        return None
+
+
+    @benchmark(benchmarking)
+    def rss_item_title(self, item):
+        """ get the item's title """
+        self.title = item.title.encode(self.encoding)
+        self.title = re.sub('(/)', '_', self.title)
+        return self.title
+
+
+    @benchmark(benchmarking)
+    def rss_item_link(self, item):
+        """ get the item's link """
+        self.link = item.link.encode(self.encoding)
+        return self.link
+
+
+    @benchmark(benchmarking)
+    def rss_item_image(self, item):
+        """ get the item's image """
+        # Search for image
+        #img_pattern = '<img src="(.*?)" align='
+        img_pattern = 'img src="(.*?)"'
         try:
-            self.rss_title = self.rss.feed.title.encode(self.encoding)
-
-            #self.rss_date = self.rss.feed.date
-        except:
-            _debug_('No title in rss feed', DINFO)
-            self.rss_title = None
-
-        try:
-            self.rss_description = self.rss.feed.description.encode(self.encoding)
-        except:
-            _debug_('No description in rss feed', DINFO)
-
-        try:
-            self.rss_image = self.rss.feed.image.url
-        except:
-            self.rss_image = None
-
-
-    def rss_count(self):
-        self.rss.count =  len(self.rss.entries)
-
-
-    def rss_item(self, item=0):
-
-        try:
-            self.title = self.rss.entries[item].title.encode(self.encoding)
-            self.title = re.sub('(/)','_',self.title)
-            description_all = self.rss.entries[item].description
-            self.link = self.rss.entries[item].link
-            # Search for image
-            #img_pattern = '<img src="(.*?)" align='
-            img_pattern = 'img src="(.*?)"'
-            try:
-                self.image = re.search(img_pattern, description_all).group(1)
-            except:
-                self.image = None
-
-        except:
-            pass
-
-        if self.link == None:
-            self.link = 'ERROR'
+            self.image = re.search(img_pattern, item.description).group(1)
+        except Exception, why:
+            self.image = None
+        return self.image
+
+
+
+class VPVideoItem(VideoItem):
+    """
+    Video podcast video item
+    """
+    @benchmark(benchmarking)
+    def __init__(self, name, url, parent):
+        """ Initialise the VPVideoItem class """
+        _debug_('VPVideoItem.__init__(name=%r, url=%r, parent=%r)' % (name, url, parent), 2)
+        VideoItem.__init__(self, name, parent)
+        self.vp_url = url
+
+
+    @benchmark(benchmarking)
+    def play(self, arg=None, menuw=None):
+        """
+        Play this Podcast
+        """
+        self.download_url = self.vp_url
+        if not os.path.exists(self.filename):
+            background = BGDownload(self.download_url, self.filename)
+            background.start()
+            popup = PopupBox(text=_('Buffering podcast...'))
+            popup.show()
+            size = 0
+            for i in range(int(config.VPODCAST_BUFFERING_TIME)):
+                if os.path.exists(self.filename):
+                    mode, ino, dev, nlink, uid, gui, size, atime, mtime, ctime = os.stat(self.filename)
+                    if size > config.VPODCAST_BUFFERING_SIZE:
+                        break
+                time.sleep(0.5)
+            else:
+                if size < 120 * 1024: # a bit arbitary, needs to be bigger than a web page
+                    popup.destroy()
+                    AlertBox(text=_('Cannot download %s') % self.filename).show()
+                    return
+            popup.destroy()
+
+        # call the play funuction of VideoItem
+        VideoItem.play(self, menuw=menuw, arg=arg)
 
 
@@ -373 +355 @@
     Download file in background
     """
+    @benchmark(benchmarking)
     def __init__(self, url, savefile):
         threading.Thread.__init__(self)
         self.url = url
         self.savefile = savefile
-
-
+        self.youtube_ie = youtube.YoutubeIE()
+        self.metacafe_ie = youtube.MetacafeIE(self.youtube_ie)
+        self.youtube_pl_ie = youtube.YoutubePlaylistIE(self.youtube_ie)
+
+
+    @benchmark(benchmarking)
     def run(self):
         try:
-            file = urllib2.urlopen(self.url)
-            info = file.info()
-            save = open(self.savefile, 'wb')
-            chunkSize = 25
-            totalBytes = int(info['Content-Length'])
-            downloadBytes = 0
-            bytesLeft = totalBytes
-            while bytesLeft > 0:
-                chunk = file.read(chunkSize)
-                readBytes = len(chunk)
-                downloadBytes += readBytes
-                bytesLeft -= readBytes
-                save.write(chunk)
-        except Exception, why:
-            _debug_('Cannot download "%s": %s' % (self.url, why), DWARNING)
+            fd = youtube.FileDownloader({
+                'usenetrc': False,
+                'username': config.YOUTUBE_USERNAME,
+                'password': config.YOUTUBE_PASSWORD,
+                'quiet': True,
+                'forceurl': False,
+                'forcetitle': False,
+                'simulate': False,
+                'format': config.YOUTUBE_FORMAT,
+                #'outtmpl': u'%(stitle)s-%(id)s.%(ext)s',
+                'outtmpl': self.savefile,
+                'ignoreerrors': False,
+                'ratelimit': None,
+                })
+            fd.add_info_extractor(self.youtube_pl_ie)
+            fd.add_info_extractor(self.metacafe_ie)
+            fd.add_info_extractor(self.youtube_ie)
+            retcode = fd.download([self.url])
+            _debug_('youtube download "%s": %s' % (self.url, retcode), DINFO)
+        except youtube.DownloadError:
+            # Not sure about this code
+            try:
+                file = urllib2.urlopen(self.url)
+                info = file.info()
+                save = open(self.savefile, 'wb')
+                chunkSize = 25
+                totalBytes = int(info['Content-Length'])
+                downloadBytes = 0
+                bytesLeft = totalBytes
+                while bytesLeft > 0:
+                    chunk = file.read(chunkSize)
+                    readBytes = len(chunk)
+                    downloadBytes += readBytes
+                    bytesLeft -= readBytes
+                    save.write(chunk)
+            except Exception, why:
+                _debug_('Cannot download "%s": %s' % (self.url, why), DWARNING)