#!/usr/bin/env python
#  *- coding: utf-8 -*
from urllib import urlopen
import re,sys,bz2,tarfile,os,commands

divq=re.compile('<div class="q">')
div=re.compile('<div')
closediv=re.compile('</div')
quote=re.compile('<div>(.*)</div>')
numpages=re.compile('div class="pager">.*<span>(\d\d\d)</span>')

def writequotes(page): 
    divbuffer=[[]]
    bufcount=0
    opencount=0
    divopen=False

    file=urlopen("http://bash.org.ru/index/%s" % page)
    for line in file.readlines():
        if divq.findall(line):
            divopen=True


        if divopen:
            opencount+=len(div.findall(line))
            divbuffer[bufcount].append(line)
            opencount-=len(closediv.findall(line))
        if opencount == 0:
            divopen=False
            divbuffer.append([])
            bufcount+=1

    buf=[]
    for l in  divbuffer:
        if l:
            try:
                str=re.sub("&quot;",'"',re.sub("&#8206;","",
                    re.sub('<br.*?>|</br.*?>','\n',re.sub('&lt;','<',re.sub('&gt;',
                        '>',quote.findall(l[7])[0])))))
                buf.append(str)
            except:
                pass


    for l in buf:
        f_koi8=open('koi8-r/fortune-bashorgru','a')
        f_uni=open('utf-8/fortune-bashorgru','a')
        if l:
            l=unicode(l,"windows-1251","replace")
            f_uni.write(unicode.encode(l+"\n%\n","utf-8","replace"))
            f_koi8.write(unicode.encode(l+"\n%\n","koi8-r","replace"))
            try:
                pass
            except:
                print l
file=urlopen("http://bash.org.ru")


for line in file.readlines():
    try:
        npages=numpages.findall(line)[0]
        break
    except:
        pass

for page in xrange(1,int(npages)+1):
    print "page %s from %s" % (page,int(npages))
    writequotes(page)

commands.getoutput("cd koi8-r; /usr/games/strfile fortune-bashorgru")
commands.getoutput("cd utf-8; /usr/games/strfile fortune-bashorgru")

tmptar_fname="tmptar.tar"
fortfile="freebsdbashorgru"

file=urlopen("http://freebsd.rusnet.org.ru/bash/fortune")
tmptar=open(tmptar_fname,"w");
tmptar.write(bz2.decompress(file.read()))
tmptar.close()
file.close()

tar=tarfile.TarFile(tmptar_fname)
tar.getnames()
os.remove(tmptar_fname)

file=open(fortfile,"w")
file.writelines(tar.extractfile("fortune.txt").readlines())

