#! /usr/bin/python3

from __future__ import print_function
from collections import defaultdict
from functools import cmp_to_key
import sys
import argparse
sys.path.append('/usr/share/botch')
from util import write_plain, read_yaml_file, cmp, parse_dose_yaml_mc
from util import read_tag_file
import os


def print_package(arg, bin2src, source=None, color=True):
    pkgname, arch, version, vpkg = arg
    sn, _ = src_of_pkg(arg, bin2src)
    if source:
        highlight = (sn == source[0])
    else:
        highlight = False
    if pkgname == "":
        pkgstring = "(*)"
        bugs = False
    elif pkgname.startswith('src:'):
        pkgname = pkgname[4:]
        bugs = srcpkgbugs.get(pkgname)
        if version:
            pkgstring = '<a title="%s" style="">src:%s</a>' % (
                version, pkgname)
        else:
            pkgstring = 'src:%s' % pkgname
    else:
        bugs = binpkgbugs.get(pkgname)
        if version:
            pkgstring = '<a title="%s:%s (= %s)">%s</a>' % (pkgname, arch,
                                                            version, pkgname)
        else:
            pkgstring = '%s:%s' % (pkgname, arch)
    if highlight:
        pkgstring = '<span style="color:#090;font-weight:bold">' + pkgstring
        pkgstring += '</span>'
    if bugs:
        bugstring = " ".join('<a href="http://bugs.debian.org/%s">%s</a>' % (b,
                                                                             b)
                             for b in bugs)
        if color:
            pkgstring = '<span style="color:#f00">%s</span> (%s)' % (pkgstring,
                                                                     bugstring)
        else:
            pkgstring = '%s (%s)' % (pkgstring, bugstring)
    if sn:
        wnpp = wnpp_stats.get(sn)
        if wnpp:
            t, i = wnpp
            pkgstring = \
                '%s (%s: <a href="http://bugs.debian.org/%s">#%s</a>)' \
                % (pkgstring, t, i, i)

    if vpkg:
        pkgstring += '<a title="%s"> → </a>' % vpkg
    return pkgstring


def by_srcname_chain(a, b):
    num_src = cmp(len(b[1]), len(a[1]))
    if num_src:
        return num_src
    cmp_src = cmp(min(a[1]), min(b[1]))
    if cmp_src:
        return cmp_src
    return cmp(a[0], b[0])


def by_affected_sources_missing(a, b):
    a1 = sum([len(v) for v in a[1].values()])
    b1 = sum([len(v) for v in b[1].values()])
    num_src = cmp(b1, a1)
    if num_src:
        return num_src
    else:
        return cmp(a[0], b[0])


def by_affected_sources_conflict(a, b):
    a1 = sum([len(v) for v in a[1].values()])
    b1 = sum([len(v) for v in b[1].values()])
    num_src = cmp(b1, a1)
    if num_src:
        return num_src
    else:
        return cmp(a[0], b[0])


def print_header(outfile, description, btsuser, btstag):
    print("""<html>
    <head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <style>
    table, th, td
    {
        border: 1px solid black;
    }
    </style>
    </head>
    <body>
    """, file=outfile)

    print(description, file=outfile)

    if btsuser and btstag:
        print("<p>Bugs are associated with packages on this page if they " +
              "carry the usertag \"%s\" of the user " % btstag +
              "\"%s\".</p>" % btsuser, file=outfile)
        print("<p>You can get an overview of all bugs tagged like that in " +
              "the <a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?t" +
              "ag=%s;users=%s\"" % (btstag, btsuser) +
              ">Debian bts</a></p>", file=outfile)
    print("<p>Hover over a package name with your cursor for architecture and "
          "version information. Hovering over the arrows in the depchain "
          "columns will show the dependency that led from one package in the "
          "chain to the next.</p>", file=outfile)


def print_missing(outfile, missing, bin2src, source=None):
    print("<h2>missing</h2>", file=outfile)

    print("<p>The packages in the third column cannot satisfy their (possibly "
          "transitive) dependencies because of the unsatisfied dependency in "
          "the last column. This is mostly because the binary package "
          "providing the dependency in the last column is Multi-Arch:no. Some "
          "of these packages need to be Multi-Arch:foreign instead. In some "
          "other cases, Build-Depends can be annotated with :native. The "
          "depchains column shows the dependency chain(s) from the packages "
          "in the third column to the unsatisfied dependency in the last "
          "column. The \"(*)\" placeholder in the depchains column represents "
          "any package in the third column. Hovering over the arrows in the "
          "depchains column with your cursor will show the dependency that "
          "led from one package in the chain to the next.</p>", file=outfile)
    print("<p>The output is first grouped by the shared unsatisfied " +
          "dependency (last column) and then by shared dependency chain " +
          "(fourth column). The groups are sorted by the number of " +
          "packages missing the dependency in the last column. Within each " +
          "group, the output is sorted by the number of packages " +
          "sharing the same dependency chain.</p>", file=outfile)
    print("<table><tr><th># of packages per missing</th><th># of packages per"
          " depchain</th><th>packages with missing (possibly transitive) depe"
          "ndencies</th><th>Depchains</th>" +
          "<th>Unsatisfied dependency</th></tr>", file=outfile)

    for missing_vpkg, v in sorted(list(missing.items()),
                                  key=cmp_to_key(by_affected_sources_missing)):
        rows = len(v)
        for c in list(v.keys()):
            if c:
                rows += len(c) - 1
        print('<tr><td rowspan="%d">%d</td>' % (rows, sum([len(s)
                                                           for s
                                                           in v.values()])),
              file=outfile)
        first = True
        for chain, v in sorted(list(v.items()),
                               key=cmp_to_key(by_srcname_chain)):
            if chain is None:
                maxnumchains = 1
            else:
                maxnumchains = len(chain)
            if chain:
                c = "".join([print_package(t, bin2src, source)
                             for t in chain[0]])
            else:
                c = ""
            if not first:
                print("<tr>", file=outfile)
            print('<td rowspan="%d">%d</td>' % (maxnumchains, len(v)) +
                  '<td rowspan="%d">%s</td><td>%s</td>'
                  % (maxnumchains, " ".join([print_package(t, bin2src, source)
                                             for t in sorted(v)]), c),
                  file=outfile)
            if first:
                print('<td rowspan="%d">%s</td></tr>'
                      % (rows, missing_vpkg),
                      file=outfile)
            else:
                print('</tr>', file=outfile)
            if chain:
                for i in range(1, len(chain)):
                    print('<tr><td>%s</td></tr>' %
                          ("".join([print_package(t, bin2src, source)
                                    for t in chain[i]])),
                          file=outfile)
            first = False
    print("</table>", file=outfile)


def print_conflict(outfile, conflict, bin2src, source=None):
    print("<h2>conflict</h2>", file=outfile)
    print("<p>The packages in the third column cannot satisfy their (possibly "
          "transitive) dependencies because the last package(s) in the first "
          "depchain have an unsatisfied conflict which is shown in the last "
          "column. The second depchain column shows the dependency chain(s) "
          "to the package which the last package(s) in the first depchain "
          "conflict with. Sometimes, multiple dependency chains sharing the "
          "same conflict exist. Hovering over the arrows in the depchains "
          "column with your cursor will show the dependency that led from one "
          "package in the chain to the next.</p>", file=outfile)
    print("<p>The output is first grouped by the shared conflicting " +
          "dependency (last column) and then by the shared dependency " +
          "chains (fourth and fifth column). The groups are sorted by the " +
          "number of packages sharing the conflict in the last " +
          "column. Within each group, the output is sorted by the number of " +
          "packages sharing the same dependency chains.</p>",
          file=outfile)
    print(
        "<table><tr><th># of packages per conflict</th><th># of packages per d"
        "epchain</th><th>packages with (possibly transitive) conflicting depen"
        "dencies</th><th>Depchain " +
        "1</th><th>Depchain2</th><th>Conflict</th></tr>", file=outfile)

    for conflict_vpkg, v in sorted(list(conflict.items()),
                                   key=cmp_to_key(
                                       by_affected_sources_conflict)):
        rows = len(v)
        for c1, c2 in list(v.keys()):
            if c1 is not None:
                c1 = len(c1)
            else:
                c1 = 0
            if c2 is not None:
                c2 = len(c2)
            else:
                c2 = 0
            if max(c1, c2) > 1:
                rows += max(c1, c2) - 1
        print('<tr><td rowspan="%d">%d</td>'
              % (rows, sum([len(s) for s in v.values()])),
              file=outfile)
        first = True
        for (chain1, chain2), v in sorted(list(v.items()),
                                          key=cmp_to_key(by_srcname_chain)):
            if chain1 is None:
                maxnumchains = len(chain2)
            elif chain2 is None:
                maxnumchains = len(chain1)
            else:
                maxnumchains = max(len(chain1), len(chain2))
            if chain1:
                c1 = "".join([print_package(t, bin2src, source)
                              for t in chain1[0]])
            else:
                c1 = ""
            if chain2:
                c2 = "".join([print_package(t, bin2src, source)
                              for t in chain2[0]])
            else:
                c2 = ""
            if not first:
                print("<tr>", file=outfile)
            print('<td rowspan="%d">%d</td>' % (maxnumchains, len(v)) +
                  '<td rowspan="%d">%s</td><td>%s</td><td>%s</td>'
                  % (maxnumchains, " ".join([print_package(t, bin2src, source)
                                             for t in sorted(v)]), c1, c2),
                  file=outfile)
            if first:
                print('<td rowspan="%d">%s</td></tr>' %
                      (rows, conflict_vpkg),
                      file=outfile)
            else:
                print('</tr>', file=outfile)
            for i in range(1, maxnumchains):
                if chain1 is not None and len(chain1) >= i + 1:
                    c1 = "".join([print_package(t, bin2src, source)
                                  for t in chain1[i]])
                else:
                    c1 = ""
                if chain2 is not None and len(chain2) >= i + 1:
                    c2 = "".join([print_package(t, bin2src, source)
                                  for t in chain2[i]])
                else:
                    c2 = ""
                print('<tr><td>%s</td><td>%s</td></tr>' %
                      (c1, c2), file=outfile)
            first = False
    print("</table>", file=outfile)


def print_footer(outfile, timestamp, srcs=[], srcsdir="", wwwroot=""):
    if srcs:
        print("<h2>Affected source packages:</h2>", file=outfile)
        srcsdir += "/"
        if srcsdir.startswith(wwwroot):
            srcsdir = srcsdir[len(wwwroot):]
        for src in sorted([s for s in srcs if s is not None]):
            print("<a href=\"%s%s.html\">%s</a>" % (srcsdir, src, src),
                  file=outfile)

    url = \
        "https://gitlab.mister-muffin.de/debian-bootstrap/bootstrap_debian_net"
    footer = """
    <hr />
    <p>The JSON data used to generate these pages was computed using botch, the
    bootstrap/build ordering tool chain. The source code of botch can be
    redistributed under the terms of the LGPL3+ with an OCaml linking
    exception. The source code can be retrieved from <a
    href="https://gitlab.mister-muffin.de/debian-bootstrap/botch">
    https://gitlab.mister-muffin.de/debian-bootstrap/botch</a></p>

    <p>The html pages were generated by code which can be retrieved from <a
    href="%s"> %s</a> and which can be redistributed under the terms of the
    AGPL3+</p>

    <p>For questions and bugreports please contact j [dot] schauer [at] email
    [dot] de.</p>
    """ % (url, url)

    if timestamp:
        print("<p>generated: %s</p>" % timestamp, file=outfile)
    print("%s</body></html>" % footer, file=outfile)


def src_of_pkg(t, bin2src):
    n, a, v, _ = t
    if n.startswith("src:"):
        return n[4:], v
    if n == "":
        return (None, None)
    res = bin2src.get((n, a, v))
    if res:
        return res
    # this can happen for a dependency on a package that is not in the archive
    print("cannot find source for %s %s" % (n, v), file=sys.stderr)
    return (None, None)


def print_top10summary(outfile, missing, conflict, bin2src, source=None):
    if not missing and not conflict:
        return

    print("<h2>Top 10 summary</h2>", file=outfile)

    print("<p>The following is a summary of the full \"missing\" and " +
          "\"conflict\" tables below. It only shows the first and last " +
          "columns of the full tables and only displays the top 10 rows.</p>",
          file=outfile)

    if missing:
        print("<h3>Missing</h3>", file=outfile)

        print("<table><tr><th># of packages per missing</th><th>Unsatisfied " +
              "dependency</th></tr>", file=outfile)
        for missing_vpkg, v in sorted(list(missing.items()),
                                      key=cmp_to_key(
                                          by_affected_sources_missing))[:10]:
            rows = len(v)
            for c in list(v.keys()):
                if c:
                    rows += len(c) - 1
            print('<tr><td>%d</td><td>%s</td></tr>' % (sum([len(s)
                                                            for s
                                                            in v.values()]),
                                                       missing_vpkg),
                  file=outfile)
        print("</table>", file=outfile)

    if conflict:
        print("<h3>Conflict</h3>", file=outfile)
        print("<table><tr><th># of packages per conflict</th>" +
              "<th>Conflict</th></tr>", file=outfile)

        for conflict_vpkg, v in sorted(list(conflict.items()),
                                       key=cmp_to_key(
                                           by_affected_sources_conflict))[:10]:
            print('<tr><td>%d</td><td>%s</td></tr>'
                  % (sum([len(s) for s in v.values()]),
                      conflict_vpkg),
                  file=outfile)
        print("</table>", file=outfile)


def create_srcpkg_pages(missing, conflict, bin2src, srcsdir, description,
                        btsuser, btstag, timestamp):
    srcsdict = defaultdict(lambda: defaultdict(lambda:
                                               defaultdict(lambda:
                                                           defaultdict(dict))))

    # now go through all missing and conflict and copy them to the source
    # packages that associate with them
    # it is not possible to find a source package for a vpkg (especially if
    # it's missing)
    for vpkg, v in missing.items():
        for chain, v in v.items():
            for c in chain:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]['missing'][vpkg][chain] = v
            for p in v:
                sn, sv = src_of_pkg(p, bin2src)
                srcsdict[sn][sv]['missing'][vpkg][chain] = v

    for vpkg, v in conflict.items():
        for (chain1, chain2), v in v.items():
            for c in chain1:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]['conflict'][vpkg][(chain1, chain2)] = v
            for c in chain2:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]['conflict'][vpkg][(chain1, chain2)] = v
            for p in v:
                sn, sv = src_of_pkg(p, bin2src)
                srcsdict[sn][sv]['conflict'][vpkg][(chain1, chain2)] = v

    if not os.path.isdir(srcsdir):
        os.mkdir(srcsdir)
    for srcpkg, versions in srcsdict.items():
        if not srcpkg:
            continue
        fname = os.path.join(srcsdir, srcpkg) + ".html"
        with open(fname, "w", encoding="utf8") as srcpkgpage:
            print_header(srcpkgpage, description, btsuser, btstag)
            for ver in versions.keys():
                print("<h1>%s</h1>" % print_package(
                    ("src:" + srcpkg, None, ver, None), bin2src, color=False),
                    file=srcpkgpage)
                print_top10summary(srcpkgpage, versions[ver].get('missing'),
                                   versions[ver].get('conflict'), bin2src,
                                   (srcpkg, ver))
                if versions[ver].get('missing'):
                    print_missing(srcpkgpage, versions[ver]['missing'],
                                  bin2src, (srcpkg, ver))
                if versions[ver].get('conflict'):
                    print_conflict(srcpkgpage, versions[ver]['conflict'],
                                   bin2src, (srcpkg, ver))
            print_footer(srcpkgpage, timestamp)

    return srcsdict.keys()


def dose2html(yamlin, outfile, description="", srcpkgbugs=set(),
              binpkgbugs=set(), btsuser=None, btstag=None, srcsdir=None,
              packages=[], timestamp=None, wwwroot=None, verbose=False):
    bin2src = dict()

    for pkg in packages:
        name = pkg['Package']
        arch = pkg['Architecture']
        ver = pkg['Version']
        src = pkg.get('Source')
        if src:
            if ' ' in src:
                srcname, srcver = src.split(' ', 1)
                srcver = srcver[1:-1]
            else:
                srcname = src
                srcver = ver
        else:
            srcname = name
            srcver = ver
        bin2src[(name, arch, ver)] = (srcname, srcver)
        bin2src[(name, arch, None)] = (srcname, None)
        provides = pkg.get('Provides')
        if not provides:
            continue
        for pname in provides.split(','):
            pname = pname.strip()
            bin2src[(pname, arch, ver)] = (srcname, srcver)
            bin2src[(pname, arch, None)] = (srcname, None)

    missing, conflict = parse_dose_yaml_mc(yamlin)

    print_header(outfile, description, btsuser, btstag)
    print_top10summary(outfile, missing, conflict, bin2src)
    print_missing(outfile, missing, bin2src)
    print_conflict(outfile, conflict, bin2src)

    if srcsdir and packages:
        srcpkgs = create_srcpkg_pages(missing, conflict, bin2src, srcsdir,
                                      description, btsuser, btstag, timestamp)
        print_footer(outfile, timestamp, srcpkgs, srcsdir, wwwroot)
    else:
        print_footer(outfile, timestamp)


class _ExtendAction(argparse.Action):
    def __init__(self, option_strings, dest, nargs=None, const=None, **kwargs):
        if nargs == 0:
            raise ValueError('nargs for extend actions must be > 0')
        if const is not None and nargs != argparse.OPTIONAL:
            raise ValueError('nargs must be %r to supply const'
                             % argparse.OPTIONAL)
        super(_ExtendAction, self).__init__(
            option_strings=option_strings, dest=dest, nargs=nargs, const=const,
            **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        items = getattr(namespace, self.dest, None)
        items = argparse._copy_items(items)
        items.extend(values)
        setattr(namespace, self.dest, items)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='given a buildcheck result, create a html overview')
    parser.register('action', 'extend', _ExtendAction)
    parser.add_argument('yamlin', type=read_yaml_file,
                        help='input in yaml format (dose3 output)')
    parser.add_argument(
        'htmlout', type=write_plain, help='output in html format')
    parser.add_argument('--desc', default="", help='descriptive HTML snippet')
    parser.add_argument('--btsuser',
                        help='bts user to associate packages with')
    parser.add_argument('--btstag',
                        help='bts tag to associate packages with')
    parser.add_argument('--srcsdir', help='output directory for ' +
                        'individual source package overviews')
    parser.add_argument('--wnpp', action='store_true',
                        help='retrieve and print wnpp status for packages')
    parser.add_argument('--packages', type=read_tag_file, action='extend',
                        default=[],
                        help='Packages file to create a mapping from binary ' +
                             'to source packages')
    parser.add_argument('--timestamp',
                        help='put a freeform timestamp string at the bottom ' +
                             'of the generated HTML')
    parser.add_argument('--wwwroot',
                        help='The HTML hyperlink to the source package '
                        'overview by default will begin with the string given '
                        'in the "--srcsdir" option. The string given by this '
                        'option will be removed from the beginning of that '
                        'path in the HTML href attribute. If this option ends '
                        'with a slash, then the resulting hyperlinks will '
                        'become relative.')
    parser.add_argument(
        '-v', '--verbose', action='store_true', help='be verbose')
    args = parser.parse_args()

    srcpkgbugs = defaultdict(list)
    binpkgbugs = defaultdict(list)

    if args.btsuser and args.btstag:
        # this only corks with python2 because SOAPpy is not available for
        # python3
        # import SOAPpy

        # server = SOAPpy.SOAPProxy('http://bugs.debian.org/cgi-bin/soap.cgi',
        #                           'Debbugs/SOAP')
        # bugnrs = server.get_usertag('debian-cross@lists.debian.org',
        #                             'cross-satisfiability')
        # buginfos = server.get_status(bugnrs['cross-satisfiability'])

        # for bug in buginfos['item']:
        #    k = bug['key']
        #    v = bug['value']
        #    if v['package']:
        #        binpkgbugs[v['package']].append(k)
        #    else:
        #        for s in v['source'].split(','):
        #            s = s.strip()
        #            srcpkgbugs[s].append(k)
        import subprocess
        res = subprocess.check_output(
            "bts select users:%s " % args.btsuser +
            "tag:%s status:open | bts status " % args.btstag +
            "fields:source,package,bug_num file:-", shell=True)
        for b in res.decode().split("\n\n"):
            if not b:
                continue
            d = {k: v for k, v in [l.split("\t")
                                   for l in b.split("\n") if l.strip()]}
            binpkgbugs[d['package']].append(d['bug_num'])
            for s in d['source'].split(','):
                s = s.strip()
                srcpkgbugs[s].append(d['bug_num'])

    wnpp_stats = dict()
    if args.wnpp and args.packages:
        import urllib.request
        url = 'https://qa.debian.org/data/bts/wnpp_rm'
        with urllib.request.urlopen(url) as f:
            # the following snippet is from
            # distro_tracker/vendor/debian/tracker_tasks.py
            for line in f:
                line = line.decode().strip()
                try:
                    package_name, wnpp_type, bug_id = \
                        line.split('|')[0].split()
                    bug_id = int(bug_id)
                except:
                    # Badly formatted
                    continue
                # Strip the colon from the end of the package name
                package_name = package_name[:-1]

                wnpp_stats[package_name] = (wnpp_type, bug_id)

    with args.htmlout as f:
        dose2html(args.yamlin, f, args.desc, srcpkgbugs, binpkgbugs,
                  args.btsuser, args.btstag, args.srcsdir, args.packages,
                  args.timestamp, args.wwwroot, args.verbose)
