#!/usr/bin/env python3
#
# List the failed, running and pending buildbot builds of a release tag.
#
# A release is built by forcing a build with the tag on the 00_force_build
# builder of the images buildbot. That builder triggers a build on every
# target builder of the release branch, which inherits the "tag" property.
# Targets which failed are triggered again the same way, one at a time.
#
# The same target builders also build the normal commits of the release
# branch, these builds have no "tag" property. The builds are therefore
# matched on the "tag" property and not on the builder name.
#
# Reading the properties of builds is very slow on the buildbot server, a
# few hundred builds already run into the gateway timeout. The "tag"
# property belongs to the build set, so it is read from the build requests
# instead, which is fast. The builds themselves are fetched without
# properties and matched to the build requests by their id.
#
# With --trigger a force build is started for every target which failed
# and was not triggered again yet. This needs the session cookie of a
# logged in buildbot user, copy the value of the TWISTED_SESSION cookie
# from the browser into the .config file next to this script:
#
#   BUILDBOT_SESSION="..."
#
# The buildbot collapses queued requests of a builder, a force build
# request which was not started yet is skipped as soon as the next one is
# queued. The force builds are therefore started one after the other, the
# next one only once the force builder is idle again.
#
# Usage: buildbot-tag-status.py [-n BUILDS] [-u URL] [-t [--dry-run]] [-r REASON] TAG

import argparse
import json
import os
import shlex
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

DEFAULT_URL = "https://buildbot.openwrt.org/images"

CONFIG_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), ".config")

# Builder which takes the forced builds and triggers the target builders,
# and the force scheduler which feeds it.
FORCE_BUILDER = "00_force_build"
FORCE_SCHEDULER = "force"

# Seconds between two checks of the force builder and the longest time to
# wait for it.
POLL_INTERVAL = 3
FORCE_TIMEOUT = 900

# Result codes of buildbot, see buildbot.process.results.
SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, CANCELLED = range(7)
RESULT_NAMES = ("success", "warnings", "failure", "skipped", "exception",
                "retry", "cancelled")

BUILD_FIELDS = ["buildid", "builderid", "buildrequestid", "number", "complete",
                "results", "state_string", "started_at", "complete_at"]
REQUEST_FIELDS = ["buildrequestid", "builderid", "claimed", "complete",
                  "results", "submitted_at", "properties"]
REQUEST_PROPERTIES = ["tag", "target"]

# Build requests fetched per API call.
REQUESTS_PER_PAGE = 2000

TIMEOUT = 120


class ApiError(Exception):
    """A request to the buildbot API failed."""

    def __init__(self, message, status=None):
        super().__init__("HTTP %s: %s" % (status, message) if status else message)
        self.status = status


def read_config(path):
    """Read the variables of a shell style KEY=value configuration file."""
    config = {}
    try:
        with open(path) as file:
            lines = file.readlines()
    except FileNotFoundError:
        return config
    except OSError as error:
        sys.exit("Could not read %s: %s" % (path, error))

    for line in lines:
        line = line.strip()
        if line.startswith("export "):
            line = line[len("export "):]
        name, sep, value = line.partition("=")
        # Comments and commands do not look like a variable name.
        if not sep or not name.isidentifier():
            continue
        try:
            config[name] = " ".join(shlex.split(value, comments=True))
        except ValueError:
            continue

    return config


def error_message(error):
    """Extract the message buildbot sends with a failed request."""
    raw = error.read()
    try:
        message = json.loads(raw).get("error")
    except (ValueError, AttributeError):
        message = raw.decode(errors="replace").strip()
    if isinstance(message, dict):
        message = message.get("message")
    return message or error.reason


def send(req):
    """Send one request to the buildbot REST API, return the decoded answer."""
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as response:
            return json.loads(response.read())
    except urllib.error.HTTPError as error:
        raise ApiError("%s: %s" % (error_message(error), req.full_url), error.code)
    except urllib.error.URLError as error:
        raise ApiError("could not reach %s: %s" % (req.full_url, error.reason))
    except (OSError, ValueError) as error:
        raise ApiError("invalid answer for %s: %s" % (req.full_url, error))


def get(base, path, query):
    """Fetch one resource of the buildbot REST API."""
    return send(urllib.request.Request(
        "%s/api/v2/%s?%s" % (base, path, urllib.parse.urlencode(query, doseq=True))))


def call(base, path, session, method, params):
    """Call a JSON-RPC method of the buildbot REST API as a logged in user."""
    body = {"id": 1, "jsonrpc": "2.0", "method": method, "params": params}
    req = urllib.request.Request("%s/api/v2/%s" % (base, path), method="POST",
                                 data=json.dumps(body).encode())
    req.add_header("Content-Type", "application/json")
    req.add_header("Accept", "application/json")
    req.add_header("Cookie", "TWISTED_SESSION=%s" % session)
    req.add_header("Origin", "%s://%s" % urllib.parse.urlsplit(base)[:2])
    req.add_header("Referer", base + "/")

    reply = send(req)
    if reply.get("error"):
        raise ApiError(reply["error"].get("message") or "unknown error")
    return reply.get("result")


def fetch_builder_names(base):
    """Map the builder ids to their names."""
    builders = get(base, "builders", {"field": ["builderid", "name"]})["builders"]
    return {builder["builderid"]: builder["name"] for builder in builders}


def fetch_builds(base, count):
    """Fetch the newest builds without their properties."""
    return get(base, "builds", {"order": "-buildid", "limit": count,
                                "field": BUILD_FIELDS})["builds"]


def fetch_requests(base, first_id):
    """Fetch the build requests starting at first_id and all unfinished ones."""
    query = {"property": REQUEST_PROPERTIES, "field": REQUEST_FIELDS}
    requests = {}

    while True:
        page = get(base, "buildrequests",
                   dict(query, buildrequestid__ge=first_id, order="buildrequestid",
                        limit=REQUESTS_PER_PAGE))["buildrequests"]
        for request in page:
            requests[request["buildrequestid"]] = request

        if len(page) < REQUESTS_PER_PAGE:
            break
        first_id = page[-1]["buildrequestid"] + 1

    # A request which waits for a long time can be older than the oldest
    # build checked, the number of unfinished requests is small.
    for request in get(base, "buildrequests",
                       dict(query, complete="false"))["buildrequests"]:
        requests[request["buildrequestid"]] = request

    return requests.values()


def fetch_request_builds(base, request_id):
    """Fetch the builds of one build request."""
    return get(base, "buildrequests/%d/builds" % request_id,
               {"field": BUILD_FIELDS})["builds"]


def prop(request, name):
    """Get the value of a property of a build request."""
    return ((request.get("properties") or {}).get(name) or [None])[0]


def current_request(requests):
    """Pick the build request which tells the current state of a builder."""
    # Buildbot merges queued requests of a builder and marks all but one of
    # them as skipped, these never get a build of their own.
    ran = [request for request in requests if request.get("results") != SKIPPED]
    return max(ran or requests, key=lambda request: request["buildrequestid"])


def state(request):
    """Get the state of a build request."""
    if not request.get("complete"):
        return "running" if request.get("claimed") else "pending"
    if request.get("results") in (SUCCESS, WARNINGS):
        return "successful"
    return "failed"


def has_failed_before(requests, current):
    """Check if an older request of a builder failed."""
    return any(state(request) == "failed" and request.get("results") != SKIPPED
               for request in requests
               if request["buildrequestid"] < current["buildrequestid"])


def result_name(results):
    """Get the name of a buildbot result code."""
    if results is not None and 0 <= results < len(RESULT_NAMES):
        return RESULT_NAMES[results]
    return "unknown"


def timestamp(seconds):
    """Format a unix time stamp in local time."""
    return time.strftime("%Y-%m-%d %H:%M", time.localtime(seconds)) if seconds else ""


def describe(base, name, request, build):
    """Build the columns describing a build request and its newest build."""
    if build:
        if build.get("complete"):
            when = build.get("complete_at")
        else:
            when = build.get("started_at")
        text = "#%d %s" % (build["number"], build.get("state_string") or "")
        url = "%s/#/builders/%d/builds/%d" % (base, build["builderid"], build["number"])
    else:
        when = request.get("submitted_at")
        text = "waiting" if state(request) == "pending" else result_name(request.get("results"))
        url = "%s/#/buildrequests/%d" % (base, request["buildrequestid"])

    return (name, timestamp(when), text, url)


def show(title, rows):
    """Print one section of the report."""
    if not rows:
        return

    print("\n%s (%d):" % (title, len(rows)))
    widths = [max(len(row[column]) for row in rows) for column in range(3)]
    for row in sorted(rows):
        print("  %-*s  %-*s  %-*s  %s" % (widths[0], row[0], widths[1], row[1],
                                          widths[2], row[2], row[3]))


def force_targets(builders):
    """Get the force build targets which rebuild the given target builders."""
    # The target builders are named <branch>_<target>.
    targets = sorted({name.split("_", 1)[1] for name in builders if "_" in name})

    # A force build triggers every builder whose name starts with the
    # target, realtek/rtl931x also rebuilds realtek/rtl931x_nand.
    return [target for target in targets
            if not any(other != target and target.startswith(other) for other in targets)]


def force_choices(base):
    """Get the values the force scheduler accepts for its list parameters."""
    scheduler = get(base, "forceschedulers/%s" % FORCE_SCHEDULER, {})["forceschedulers"][0]

    choices = {}
    fields = list(scheduler.get("all_fields") or [])
    while fields:
        field = fields.pop()
        fields.extend(field.get("fields") or [])
        if field.get("choices") is not None:
            choices[field.get("fullName") or field.get("name")] = set(field["choices"])
    return choices


def wait_for(check, what):
    """Call check until it returns something, give up after FORCE_TIMEOUT."""
    deadline = time.monotonic() + FORCE_TIMEOUT
    while True:
        result = check()
        if result:
            return result
        if time.monotonic() > deadline:
            raise ApiError("gave up waiting for %s" % what)
        time.sleep(POLL_INTERVAL)


def force_builder_idle(base, builderid):
    """Check that no request is queued at the force builder."""
    # Buildbot only filters on fields which are selected.
    return not get(base, "builders/%d/buildrequests" % builderid,
                   {"claimed": "false", "complete": "false",
                    "field": ["buildrequestid", "claimed", "complete"]})["buildrequests"]


def finished_request(base, request_id):
    """Get a build request once it is complete."""
    request = get(base, "buildrequests/%d" % request_id,
                  {"field": REQUEST_FIELDS})["buildrequests"][0]
    return request if request.get("complete") else None


def force_build(base, session, builderid, target, tag, reason):
    """Queue a force build of one target, return its build request id."""
    result = call(base, "forceschedulers/%s" % FORCE_SCHEDULER, session, "force",
                  {"builderid": str(builderid), "username": "", "reason": reason,
                   "priority": 0, "branch": "", "project": "", "repository": "",
                   "revision": "", "target": target, "tag": tag})

    # The result is the build set id and the build request id per builder.
    try:
        return int(next(iter(result[1].values())))
    except (TypeError, IndexError, KeyError, StopIteration, ValueError):
        raise ApiError("unexpected answer: %r" % (result,))


def trigger_rebuilds(base, session, builderid, targets, tag, reason):
    """Force a build of every target, one after the other."""
    failed = 0

    for index, target in enumerate(targets, 1):
        print("[%d/%d] %s ... " % (index, len(targets), target), end="", flush=True)

        try:
            # A queued request would be collapsed with this one, wait
            # until the force builder took it.
            wait_for(lambda: force_builder_idle(base, builderid), "the force builder")
            request_id = force_build(base, session, builderid, target, tag, reason)
            request = wait_for(lambda: finished_request(base, request_id),
                               "build request %d" % request_id)
        except ApiError as error:
            print("FAILED: %s" % error, flush=True)
            if error.status in (401, 403):
                sys.exit("The session is not allowed to force builds, update "
                         "BUILDBOT_SESSION in %s." % CONFIG_FILE)
            failed += 1
            continue

        url = "%s/#/buildrequests/%d" % (base, request_id)
        if request.get("results") in (SUCCESS, WARNINGS):
            print("triggered %s" % url, flush=True)
        else:
            print("FAILED: force build %s %s" % (result_name(request.get("results")), url),
                  flush=True)
            failed += 1

    return failed


def main():
    parser = argparse.ArgumentParser(
        description="List the failed, running and pending buildbot builds of a release tag.")
    parser.add_argument("tag", metavar="TAG",
                        help="release tag to check, for example v25.12.6")
    parser.add_argument("-n", "--builds", type=int, default=2000, metavar="BUILDS",
                        help="number of the newest builds to check (default: %(default)s)")
    parser.add_argument("-u", "--url", default=DEFAULT_URL,
                        help="buildbot to check (default: %(default)s)")
    parser.add_argument("-t", "--trigger", action="store_true",
                        help="force a build of every target which failed")
    parser.add_argument("--dry-run", action="store_true",
                        help="only list the targets --trigger would force a build of")
    parser.add_argument("-r", "--reason", default="Trigger rebuild",
                        help="reason given for the force builds (default: %(default)s)")
    args = parser.parse_args()

    if args.builds < 1:
        sys.exit("The number of builds has to be positive.")
    base = args.url.rstrip("/")

    session = None
    if args.trigger and not args.dry_run:
        session = read_config(CONFIG_FILE).get("BUILDBOT_SESSION")
        if not session:
            sys.exit("Set BUILDBOT_SESSION in %s to the value of the TWISTED_SESSION "
                     "cookie of a logged in buildbot session." % CONFIG_FILE)

    try:
        names = fetch_builder_names(base)
        builds = fetch_builds(base, args.builds)
        if not builds:
            sys.exit("The buildbot did not return any build.")
        oldest = min(builds, key=lambda build: build["buildid"])
        requests = fetch_requests(base, min(build["buildrequestid"] for build in builds))
    except ApiError as error:
        sys.exit("Could not read the builds: %s" % error)

    # A retried build request has more than one build, the newest one counts.
    newest_build = {}
    for build in sorted(builds, key=lambda build: build["buildid"]):
        newest_build[build["buildrequestid"]] = build

    # Group the requests of the tag per target builder. The force builder
    # serves all targets, so its requests are kept apart per target.
    jobs = {}
    for request in requests:
        if prop(request, "tag") != args.tag:
            continue

        name = names.get(request["builderid"], "builder %d" % request["builderid"])
        if name == FORCE_BUILDER:
            name = "%s %s" % (FORCE_BUILDER, prop(request, "target") or "")
        jobs.setdefault(name, []).append(request)

    print("Checked %d builds started since %s."
          % (len(builds), timestamp(oldest.get("started_at"))))
    if not jobs:
        print("No build of tag %s found, try to check more builds with --builds."
              % args.tag)
        return 0

    sections = {"failed": [], "running": [], "pending": [], "triggers": []}
    builders = successful = rebuilt = 0
    failed_builders = []

    for name, job in jobs.items():
        request = current_request(job)
        current = state(request)

        build = newest_build.get(request["buildrequestid"])
        if not build and request.get("claimed") and request.get("results") != SKIPPED:
            # The build started before the oldest build checked, a skipped
            # request is claimed as well but never gets a build.
            try:
                found = fetch_request_builds(base, request["buildrequestid"])
            except ApiError as error:
                sys.exit("Could not read the builds of request %d: %s"
                         % (request["buildrequestid"], error))
            build = max(found, key=lambda build: build["buildid"], default=None)

        row = describe(base, name, request, build)

        if name.startswith(FORCE_BUILDER + " "):
            # A skipped or failed force build never triggered its target.
            if current == "failed":
                sections["triggers"].append(row)
            elif current != "successful":
                sections[current].append(row)
            continue

        builders += 1
        if current == "successful":
            successful += 1
            rebuilt += has_failed_before(job, request)
        else:
            sections[current].append(row)
        if current == "failed":
            failed_builders.append(name)

    print("Tag %s: %d builders, %d successful (%d after a rebuild), %d failed, "
          "%d running, %d pending" % (args.tag, builders, successful, rebuilt,
                                      len(sections["failed"]), len(sections["running"]),
                                      len(sections["pending"])))

    show("Failed builds", sections["failed"])
    show("Running builds", sections["running"])
    show("Pending build requests", sections["pending"])
    show("Force builds which did not trigger their target", sections["triggers"])

    if not (args.trigger or args.dry_run):
        return 0

    targets = force_targets(failed_builders)
    if not targets:
        print("\nNo failed target to force a build of.")
        return 0

    force_id = next((builderid for builderid, name in names.items()
                     if name == FORCE_BUILDER), None)
    if force_id is None:
        sys.exit("The buildbot has no %s builder." % FORCE_BUILDER)

    try:
        choices = force_choices(base)
    except ApiError as error:
        sys.exit("Could not read the force scheduler: %s" % error)

    if args.tag not in choices.get("tag", {args.tag}):
        sys.exit("The force scheduler does not offer tag %s." % args.tag)
    unknown = [target for target in targets if target not in choices.get("target", targets)]
    if unknown:
        print("\nThe force scheduler does not offer the target(s) %s, skipping them."
              % ", ".join(unknown), file=sys.stderr)
        targets = [target for target in targets if target not in unknown]

    if args.dry_run:
        print("\nWould force builds of %d target(s): %s" % (len(targets), " ".join(targets)))
        return 0

    print("\nForcing builds of %d target(s) for %s:" % (len(targets), args.tag))
    failed = trigger_rebuilds(base, session, force_id, targets, args.tag, args.reason)
    if failed:
        print("%d of %d force build(s) failed." % (failed, len(targets)), file=sys.stderr)
        return 1

    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(130)