#!/usr/bin/python -OO
# Copyright 2008-2010 The SABnzbd-Team <team@sabnzbd.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

"""
sabnzbd.urlgrabber - Queue for grabbing NZB files from websites
"""

import os
import time
import re
import logging
import Queue
import urllib
from threading import *

import socket
try:
    socket.ssl
    _PROTOCOL = 'https'
except:
    _PROTOCOL = 'http'

import sabnzbd
import sabnzbd.misc as misc
import sabnzbd.dirscanner as dirscanner
import sabnzbd.nzbqueue as nzbqueue
import sabnzbd.cfg as cfg
from sabnzbd.lang import Ta

#------------------------------------------------------------------------------
# Wrapper functions

__GRABBER = None  # Global pointer to url-grabber instance

def init():
    global __GRABBER
    if __GRABBER:
        __GRABBER.__init__()
    else:
        __GRABBER = URLGrabber()

def start():
    global __GRABBER
    if __GRABBER: __GRABBER.start()

def add(url, future_nzo):
    global __GRABBER
    if __GRABBER: __GRABBER.add(url, future_nzo)

def stop():
    global __GRABBER
    if __GRABBER:
        __GRABBER.stop()
        try:
            __GRABBER.join()
        except:
            pass

def alive():
    global __GRABBER
    if __GRABBER:
        return __GRABBER.isAlive()
    else:
        return False


#------------------------------------------------------------------------------
_RETRIES = 10

class URLGrabber(Thread):
    def __init__(self):
        Thread.__init__(self)
        self.queue = Queue.Queue()
        for tup in sabnzbd.nzbqueue.get_urls():
            url, nzo = tup
            self.queue.put((url, nzo, _RETRIES))
        self.shutdown = False

    def add(self, url, future_nzo):
        """ Add an URL to the URLGrabber queue """
        self.queue.put((url, future_nzo, _RETRIES))

    def stop(self):
        logging.info('URLGrabber shutting down')
        self.shutdown = True
        self.queue.put((None, None, 0))

    def run(self):
        logging.info('URLGrabber starting up')
        self.shutdown = False

        while not self.shutdown:
            (url, future_nzo, retry_count) = self.queue.get()
            if not url:
                continue

            try:
                # If nzo entry deleted, give up
                try:
                    deleted = future_nzo.deleted
                except:
                    deleted = True
                if deleted:
                    logging.debug('Dropping URL %s, job entry missing', url)
                    continue

                # Add nzbmatrix credentials if needed
                url, matrix_id = _matrix_url(url)

                # _grab_url cannot reside in a function, because the tempfile
                # would not survive the end of the function
                logging.info('Grabbing URL %s', url)
                opener = urllib.FancyURLopener({})
                opener.prompt_user_passwd = None
                opener.addheaders = []
                opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__)
                opener.addheader('Accept-encoding','gzip')
                filename = None
                category = None
                try:
                    fn, header = opener.retrieve(url)
                except:
                    fn = None

                if fn:
                    for tup in header.items():
                        try:
                            item = tup[0].lower()
                            value = tup[1].strip()
                        except:
                            continue
                        if item in ('category_id', 'x-dnzb-category'):
                            category = value
                        elif item in ('x-dnzb-moreinfo',):
                            more_info = value
                        elif item in ('x-dnzb-name',):
                            filename = value
                            if not filename.endswith('.nzb'):
                                filename += '.nzb'

                        if not filename:
                            for item in tup:
                                if "filename=" in item:
                                    filename = item[item.index("filename=") + 9:].strip(';').strip('"')

                if matrix_id:
                    fn, msg, retry = _analyse_matrix(fn, matrix_id)
                    category = _MATRIX_MAP.get(category, category)
                else:
                    msg = ''
                    retry = True

                # Check if the filepath is specified, if not, check if a retry is allowed.
                if not fn:
                    retry_count -= 1
                    if retry_count > 0 and retry:
                        logging.info('Retry URL %s', url)
                        self.queue.put((url, future_nzo, retry_count))
                    else:
                        misc.bad_fetch(future_nzo, url, msg, retry=True)
                    continue

                if not filename:
                    filename = os.path.basename(url) + '.nzb'
                filename = misc.sanitize_foldername(filename)
                _r, _u, _d = future_nzo.get_repair_opts()
                pp = sabnzbd.opts_to_pp(_r, _u, _d)
                script = future_nzo.get_script()
                cat = future_nzo.get_cat()
                if cat is None and category:
                    cat = misc.CatConvert(category)
                priority = future_nzo.get_priority()
                nzbname = future_nzo.get_dirname_rename()

                # Check if nzb file
                if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'):
                    res = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname)
                    if res == 0:
                        nzbqueue.remove_nzo(future_nzo.nzo_id, add_to_history=False, unload=True)
                    elif res == -2:
                        self.add(url, future_nzo)
                    elif matrix_id:
                        # Keep retrying NzbMatrix forever
                        self.add(url, future_nzo)
                    else:
                        misc.bad_fetch(future_nzo, url, retry=True, content=True)
                # Check if a supported archive
                else:
                    if dirscanner.ProcessArchiveFile(filename, fn, pp, script, cat, priority=priority) == 0:
                        nzbqueue.remove_nzo(future_nzo.nzo_id, add_to_history=False, unload=True)
                    else:
                        # Not a supported filetype, not an nzb (text/html ect)
                        try:
                            os.remove(fn)
                        except:
                            pass
                        misc.bad_fetch(future_nzo, url, retry=True, content=True)
            except:
               logging.error('URLGRABBER CRASHED', exc_info = True)
               logging.info("URLGRABBER Traceback: ", exc_info = True)

            # Don't pound the website!
            time.sleep(5.0)



#-------------------------------------------------------------------------------
_RE_NZBMATRIX = re.compile(r'nzbmatrix.com/(.*)[\?&]id=(\d+)', re.I)
_RE_NZBMATRIX_USER = re.compile(r'&username=([^&=]+)', re.I)
_RE_NZBMATRIX_API  = re.compile(r'&apikey=([^&=]+)', re.I)

def _matrix_url(url):
    """ Patch up the url for nzbmatrix.com """

    matrix_id = 0
    m = _RE_NZBMATRIX.search(url)
    if m:
        matrix_id = m.group(2)
        if not _RE_NZBMATRIX_USER.search(url) or not _RE_NZBMATRIX_API.search(url):
            user = urllib.quote_plus(cfg.MATRIX_USERNAME.get())
            key = urllib.quote_plus(cfg.MATRIX_APIKEY.get())
            url = '%s://api.nzbmatrix.com/v1.1/download.php?id=%s&username=%s&apikey=%s' % \
                  (_PROTOCOL, matrix_id, user, key)
    return url, matrix_id

_RE_MATRIX_ERR = re.compile(r'please_wait[_ ]+(\d+)', re.I)

def _analyse_matrix(fn, matrix_id):
    """ Analyse respons of nzbmatrix
    """
    msg = ''
    if not fn:
        # No response, just retry
        return (None, msg, True)
    try:
        f = open(fn, 'r')
        data = f.read(40)
        f.close()
    except:
        return (None, msg, True)

    # Check for an error response
    if data and data.startswith('error'):
        # Check if we are required to wait - if so sleep the urlgrabber
        m = _RE_MATRIX_ERR.search(data)
        if m:
            wait = int(m.group(1))
            if wait:
                logging.debug('Sleeping URL grabber %s sec', wait)
                time.sleep(min(wait, 60))
                # Return, but tell the urlgrabber to retry
                return (None, msg, True)
        else:
            # Clear error message, don't retry
            msg = Ta('warn-matrixFail@1') % data
            return (None, msg, False)

    if data.startswith("<!DOCTYPE"):
        # We got HTML, probably a temporary problem, keep trying
        msg = Ta('warn-matrixBadRep@1') % matrix_id
        return (None, msg, True)

    return fn, msg, False



#------------------------------------------------------------------------------
_MATRIX_MAP = {
'28' : 'anime.all',
'20' : 'apps.linux',
'19' : 'apps.mac',
'21' : 'apps.other',
'18' : 'apps.pc',
'55' : 'apps.phone',
'52' : 'apps.portable',
'53' : 'documentaries.hd',
'9'  : 'documentaries.std',
'16' : 'games.dreamcast',
'45' : 'games.ds',
'46' : 'games.gamecube',
'17' : 'games.other',
'10' : 'games.pc',
'15' : 'games.ps1',
'11' : 'games.ps2',
'43' : 'games.ps3',
'12' : 'games.psp',
'44' : 'games.wii',
'51' : 'games.wii vc',
'13' : 'games.xbox',
'14' : 'games.xbox360',
'56' : 'games.xbox360 (other)',
'54' : 'movies.brrip',
'2'  : 'movies.divx/xvid',
'1'  : 'movies.dvd',
'50' : 'movies.hd (image)',
'42' : 'movies.hd (x264)',
'4'  : 'movies.other',
'3'  : 'movies.svcd/vcd',
'48' : 'movies.wmv-hd',
'24' : 'music.dvd',
'23' : 'music.lossless',
'22' : 'music.mp3, albums',
'47' : 'music.mp3, singles',
'27' : 'music.other',
'25' : 'music.video',
'49' : 'other.audio, books',
'36' : 'other.e-books',
'33' : 'other.emulation',
'39' : 'other.extra, pars/fills',
'37' : 'other.images',
'38' : 'other.mobile, phone',
'40' : 'other.other',
'34' : 'other.ppc/pda',
'26' : 'other.radio',
'6'  : 'tv.divx/xvid',
'5'  : 'tv.dvd',
'41' : 'tv.hd',
'8'  : 'tv.other',
'7'  : 'tv.sport/ent'
}
