# Part of flinks
# (C) Martin Bays 2008
# Released under the terms of the GPLv3

import sys, os
from string import *

import re
import codecs

from subprocess import Popen, PIPE

from .constants import USER_AGENT
from .readme import README

def lynxDump(url, lynxArgs=[]):
    if url == "special:README":
        return README, [], ''

    try:
        p = Popen(['lynx', '-dump', '-force_html', '-useragent="%s via lynx"' % USER_AGENT] +
                lynxArgs + [url],
                stdin=None, stdout=PIPE, stderr=PIPE)
        (lynxStdout, lynxErrout) = (p.stdout, p.stderr)
    except OSError:
        return "", [], "Fatal error - lynx execution failed. Is it installed?"

    # TODO: work out the encoding somehow? For now we assume it's latin1...
    lynxDecoded = codecs.EncodedFile(lynxStdout, 'utf8', 'latin1', 'replace')

    dumped = ''
    refdumped = ''
    linkUrls = []
    readingRefs = False
    for binaryline in lynxDecoded:
        line = lynxDecoded.decode(binaryline)[0]
        if line == 'References\n':
            if readingRefs:
                # The previous matched 'References' was part of the
                # document...
                dumped += refdumped
                refdumped = ''
                linkUrls = []
            readingRefs = True

        if readingRefs:
            m = re.match(r'\s*\d+\. (.*)\n', line)
            if m:
                linkUrls += [m.groups()[0]]
            refdumped += line
        else:
            dumped += line

    lynxDecoded.close()
    lynxErr = lynxErrout.read()
    return dumped, linkUrls, lynxErr
