import re
import feedparser
import urllib
import sys
from datetime import *

from django.core.management.base import BaseCommand, CommandError
from django.utils.html import strip_tags
from feeds.models import *

# see Command definition at the end

# TODO: need a u.encode('utf-8') somewhere in here...

# this mechanism isn't robust yet b/c any small sets get parsed repeatedly
MIN_SET_SIZE = 3   # Need to find at least this many images for each set

###############################################################################
def scrape_pics_from_html(pset, html):
    
    if(type(pset) != PicSet):
        raise("Type error, expected PicSet")

    # assumptions: one <img> per line, ordering, etc
    if(pset.feed.shortname == "vectortut"):
        mset = re.finditer('<img src="(http://\S+cloudfront.net\S+)" border=', html)
    elif(pset.feed.shortname == "nzinzi"):
        mset = re.finditer('src=[\'\"](http://ntamak.free.fr/\S+/\S+)[\'\"]', html)
    else:
        mset = re.finditer('<img src="(http://\S+)" \S* width="(\d+)" height="(\d+)"', html)

    pics = list()
    index = 0
    for m in mset:
        index += 1
        p = Pic()
        #print m.group(1)
        #print m.group(2)
        # This one line strips any paren content; disabled because it's just the
        # set title repeated
        #p.title = m.group(2).split(" (")[0]
        p.title = "#" + str(index)
        p.set = pset
        # empty string isn't the same as leaving this null
        #p.caption = ""
        p.original_url = m.group(1)
        p.source_url = pset.source_url
        if pset.feed.shortname not in ("vectortut", "nzinzi"): 
            p.width = m.group(2)
            p.height = m.group(3)
        pics.append(p)

    if(len(pics) < MIN_SET_SIZE):
        print "Didn't find enough pictures to save this set (found " + \
            str(len(pics)) + ", MIN_SET_SIZE=" + str(MIN_SET_SIZE) + ")"
        pset.is_valid = False
        pset.save()
        return

    # TODO: oh boy, serial, this is a horrible way to do it! 
    for p in pics:
        p.save()

    # add a thumbnail image for the picset (TODO: resize thumbnail?)
    if not pset.image:
        pset.image = pics[0]
        pset.save()

    print "Found and saved " + str(len(pics)) + " new Pics"

###############################################################################
def scrape_pics_bypage(pset):
    
    if(type(pset) != PicSet):
        raise("Type error, expected PicSet")

    #print "DEBUG"
    #html = open("./19340-evolution-16-pics.html")
    print "Scraping from " + pset.source_url + "..."
    html = urllib.urlopen(pset.source_url)

    if(pset.feed.shortname == "acidcow"):
        mset = re.finditer('src="(http://acidcow.com/pics/\d+/\S+)" alt=\'([^\.]+)\' title', html.read())
    elif(pset.feed.shortname == "jetfunny"):
        mset = re.finditer('src=\'(http://jetfunnypictures.com/img/jetfunnypictures.com/\S+)\'', html.read())
    else:
        mset = re.finditer('src=[\'\"](http://\S+)[\"\']', html.read())

    pics = list()
    index = 0
    for m in mset:
        index += 1
        p = Pic()
        #print m.group(1)
        #print m.group(2)
        # This oneline strips any paren content; disabled because it's just the
        # set title repeated
        #p.title = m.group(2).split(" (")[0]
        p.title = "#" + str(index)
        p.set = pset
        p.caption = ""
        p.original_url = m.group(1)
        p.source_url = pset.source_url
        pics.append(p)

    # close the connection
    html.close()

    if(len(pics) < MIN_SET_SIZE):
        print "Didn't find enough pictures to save this set (found " + \
            str(len(pics)) + ", MIN_SET_SIZE=" + str(MIN_SET_SIZE) + ")"
        pset.is_valid = False
        pset.save()
        return

    # TODO: oh boy, serial, this is a horrible way to do it! 
    for p in pics:
        p.save()

    # add a thumbnail image for the picset (TODO: resize thumbnail?)
    if not pset.image:
        pset.image = pics[0]
        pset.save()

    print "Found and saved " + str(len(pics)) + " new Pics"


###############################################################################
def scrape_feed(feed_shortname):

    try:
        feed = PicFeed.objects.get(shortname=feed_shortname)
    except Exception as e:
        sys.stderr.write("Error finding feed by shortname: " + \
            feed_shortname + "\n")
        raise e

    print "Fetching feed for " + feed_shortname + ": " + feed.rssfeed_url
    f = feedparser.parse(feed.rssfeed_url)
    #print "DEBUG"
    #f = feedparser.parse("./acidcow_com?format=xml")
    #f = feedparser.parse("./ButDoesItFloat?format=xml")

    psets = list()
    for e in f['entries']:
        pset = dict()
        if(feed_shortname == "acidcow"):
            pset['source_url'] = e.guid
        elif(feed_shortname == "butdoesitfloat"):
            pset['source_url'] = e.comments
        else:
            pset['source_url'] = e.link
        pset['created'] = e.date
        pset['title'] = e.title
        if(feed_shortname == "nzinzi"):
            pset['category'] = "Art"
        else:
            pset['category'] = e.category
        pset['description'] = e.description
        psets.append(pset)

    # clean up parser
    del f
    
    if(feed_shortname == u"acidcow"):
        psets = filter(lambda s: s['category'] in (u'Pics', u'Picdump',u'Celebs',u'Girls',u'Cars'), psets)
    elif(feed_shortname == u"vectortut"):
        psets = filter(lambda s: s['category'] in (u'Inspirational'), psets)

    new_psets = filter(lambda s: \
        0 == len(PicSet.objects.filter(source_url=s['source_url'])), \
        psets)

    for pset in new_psets:
        print "Parsing PicSet: " + pset['title'] + " (" + pset['category'] + ")"
        p = PicSet()
        p.feed = feed
        p.source_url = pset['source_url']
        if(feed_shortname == "nzinzi"):
            p.created = datetime.strptime(pset['created'][:-10], \
                "%Y-%m-%dT%H:%M:%S")
        else:
            p.created = datetime.strptime(pset['created'][:-6], \
                "%a, %d %b %Y %H:%M:%S")
        # This oneline strips any paren content
        p.title = pset['title'].split(" (")[0]
        if(feed_shortname in ("butdoesitfloat", "vectortut", "nzinzi")):
            # Category is ignored for these sites
            p.category = Category.objects.get_or_create(name="Art")[0]
        else:
            p.category = Category.objects.get_or_create(name=pset['category'])[0]

        # Ok, this is where we split out and do custom, per-site processing
        if(feed_shortname == u"acidcow"):
            if(len(pset['description']) > 0):
                p.description = strip_tags(pset['description'])
                if(p.description.startswith(u"Similar posts:")):
                    p.description = ' '
            p.save()
            print "Great, saved: " + p.title + " (id=" + str(p.id) + ")"
            scrape_pics_bypage(p)
        elif(feed_shortname == u"butdoesitfloat"):
            # the descriptions here are usually important
            # TODO: should comment URL instead of link url (which bounces)?
            p.description = pset['description'].split("<img")[0]
            p.save()
            print "Great, saved: " + p.title + " (id=" + str(p.id) + ")"
            scrape_pics_from_html(p, pset['description'])
        elif(feed_shortname == u"vectortut"):
            p.description = ""
            p.save()
            print "Great, saved: " + p.title + " (id=" + str(p.id) + ")"
            scrape_pics_from_html(p, pset['description'])
        elif(feed_shortname == u"jetfunny"):
            if(len(pset['description']) > 0):
                p.description = strip_tags(pset['description'].split("</p>")[0])
            p.save()
            print "Great, saved: " + p.title + " (id=" + str(p.id) + ")"
            scrape_pics_bypage(p)
        elif(feed_shortname == u"nzinzi"):
            p.description = ""
            p.save()
            print "Great, saved: " + p.title + " (id=" + str(p.id) + ")"
            scrape_pics_from_html(p, pset['description'])
        else:
            print "ERROR: unknown PicFeed: " + pset['feed']
        #parse_page(s['source_url'], httplib.HTTPConnection)
        #break # DEBUG


###############################################################################

class Command(BaseCommand):
    args = '<feed_shortname feed_shortname ...>'
    help = 'Fetches RSS, parses, and possibly scrapes HTML for the given feeds/sources'

    def handle(self, *args, **options):
        if len(args) < 1:
            sys.stderr.write("Need to specify at least one <feed_shortname>...\n")
            return

        for shortname in args:
            try:
                scrape_feed(shortname)
            except Exception as e:
                sys.stderr.write("Error scraping " + shortname + ":\n")
                sys.stderr.write(str(e) + "\n")
                #raise e

        sys.stdout.flush()
        sys.stdout.write('Done scraping feeds.\n')