From 27f0b7fe926d70c4f3e9c5bb1b297b76d6e1f704 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Petr=20=C5=A0tetiar?= Date: Tue, 16 Mar 2021 18:21:15 +0100 Subject: [PATCH] phase1,phase2: implement round robin builds MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Gather newest complete and not skipped build timestamps, reverse sort them and use that as builder priority, so the newest built targets are at the bottom. Signed-off-by: Petr Å tetiar --- phase1/master.cfg | 81 ++++++++++++++++++++++++++++++++++++++++++----- phase2/master.cfg | 71 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 140 insertions(+), 12 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 2ccac8e..53d1812 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -4,13 +4,17 @@ import os import re import base64 -import random import subprocess import configparser -from datetime import timedelta +from dateutil.tz import tzutc +from datetime import datetime, timedelta + +from twisted.internet import defer +from twisted.python import log from buildbot import locks +from buildbot.data import resultspec from buildbot.changes import filter from buildbot.changes.gitpoller import GitPoller from buildbot.config import BuilderConfig @@ -19,6 +23,7 @@ from buildbot.plugins import schedulers from buildbot.plugins import steps from buildbot.plugins import util from buildbot.process import properties +from buildbot.process import results from buildbot.process.factory import BuildFactory from buildbot.process.properties import Interpolate from buildbot.process.properties import Property @@ -123,6 +128,69 @@ c['configurators'] = [util.JanitorConfigurator( hour=6, )] +@defer.inlineCallbacks +def getNewestCompleteTime(bldr): + """Returns the complete_at of the latest completed and not SKIPPED + build request for this builder, or None if there are no such build + requests. We need to filter out SKIPPED requests because we're + using collapseRequests=True which is unfortunately marking all + previous requests as complete when new buildset is created. + + @returns: datetime instance or None, via Deferred + """ + + bldrid = yield bldr.getBuilderId() + completed = yield bldr.master.data.get( + ('builders', bldrid, 'buildrequests'), + [ + resultspec.Filter('complete', 'eq', [True]), + resultspec.Filter('results', 'ne', [results.SKIPPED]), + ], + order=['-complete_at'], limit=1) + if not completed: + return + + return completed[0]['complete_at'] + +@defer.inlineCallbacks +def prioritizeBuilders(master, builders): + """Returns sorted list of builders by their last timestamp of completed and + not skipped build. + + @returns: list of sorted builders + """ + + def is_building(bldr): + return bool(bldr.building) or bool(bldr.old_building) + + def bldr_info(bldr): + d = defer.maybeDeferred(getNewestCompleteTime, bldr) + d.addCallback(lambda complete_at: (complete_at, bldr)) + return d + + def bldr_sort(item): + (complete_at, bldr) = item + + if not complete_at: + date = datetime.min + complete_at = date.replace(tzinfo=tzutc()) + + if is_building(bldr): + date = datetime.max + complete_at = date.replace(tzinfo=tzutc()) + + return (complete_at, bldr.name) + + results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders]) + results.sort(key=bldr_sort) + + for r in results: + log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0])) + + return [r[1] for r in results] + +c['prioritizeBuilders'] = prioritizeBuilders + ####### CHANGESOURCES work_dir = os.path.abspath(ini.get("general", "workdir") or ".") @@ -456,13 +524,10 @@ def GetNextBuild(builder, requests): for r in requests: if r.properties and r.properties.hasProperty("tag"): return r - return requests[0] -def prioritizeBuilders(buildmaster, builders): - random.shuffle(builders) - return builders - -c['prioritizeBuilders'] = prioritizeBuilders + r = requests[0] + log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid)) + return r def MakeEnv(overrides=None, tryccache=False): env = { diff --git a/phase2/master.cfg b/phase2/master.cfg index ac44fcd..a54c191 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -8,15 +8,21 @@ import random import subprocess import configparser -from datetime import timedelta +from dateutil.tz import tzutc +from datetime import datetime, timedelta + +from twisted.internet import defer +from twisted.python import log from buildbot import locks +from buildbot.data import resultspec from buildbot.changes import filter from buildbot.changes.gitpoller import GitPoller from buildbot.config import BuilderConfig from buildbot.plugins import schedulers from buildbot.plugins import steps from buildbot.plugins import util +from buildbot.process import results from buildbot.process.factory import BuildFactory from buildbot.process.properties import Property from buildbot.process.properties import WithProperties @@ -319,9 +325,66 @@ def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): def IsSharedWorkdir(step): return bool(step.getProperty("shared_wd")) -def prioritizeBuilders(buildmaster, builders): - random.shuffle(builders) - return builders +@defer.inlineCallbacks +def getNewestCompleteTime(bldr): + """Returns the complete_at of the latest completed and not SKIPPED + build request for this builder, or None if there are no such build + requests. We need to filter out SKIPPED requests because we're + using collapseRequests=True which is unfortunately marking all + previous requests as complete when new buildset is created. + + @returns: datetime instance or None, via Deferred + """ + + bldrid = yield bldr.getBuilderId() + completed = yield bldr.master.data.get( + ('builders', bldrid, 'buildrequests'), + [ + resultspec.Filter('complete', 'eq', [True]), + resultspec.Filter('results', 'ne', [results.SKIPPED]), + ], + order=['-complete_at'], limit=1) + if not completed: + return + + return completed[0]['complete_at'] + +@defer.inlineCallbacks +def prioritizeBuilders(master, builders): + """Returns sorted list of builders by their last timestamp of completed and + not skipped build. + + @returns: list of sorted builders + """ + + def is_building(bldr): + return bool(bldr.building) or bool(bldr.old_building) + + def bldr_info(bldr): + d = defer.maybeDeferred(getNewestCompleteTime, bldr) + d.addCallback(lambda complete_at: (complete_at, bldr)) + return d + + def bldr_sort(item): + (complete_at, bldr) = item + + if not complete_at: + date = datetime.min + complete_at = date.replace(tzinfo=tzutc()) + + if is_building(bldr): + date = datetime.max + complete_at = date.replace(tzinfo=tzutc()) + + return (complete_at, bldr.name) + + results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders]) + results.sort(key=bldr_sort) + + for r in results: + log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0])) + + return [r[1] for r in results] c['prioritizeBuilders'] = prioritizeBuilders c['builders'] = [] -- 2.30.2