From a3fad85cce070c0f8a296d4fdb35926e67ea7b6d Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Tue, 18 Dec 2012 20:06:39 -0800 Subject: [PATCH] compat: rewrite ckmake in Python This rewrites ckmake in Python. I suspected that we can still improve compilation down by making ckmake multithreaded. I was right, and in order to make this multithreaded I picked python and ncurses to display results. This shaves down 6 minutes for compilation of compat-drivers on 24 kernels from 25 minutes down to 19 minutes. This can likely be improved further. Before: real 25m28.705s user 506m26.003s sys 69m45.990s After: real 19m4.757s user 486m26.236s sys 70m5.579s Signed-off-by: Luis R. Rodriguez --- bin/ckmake | 582 +++++++++++++++++++++++++++++------------------------ 1 file changed, 321 insertions(+), 261 deletions(-) diff --git a/bin/ckmake b/bin/ckmake index 5fd3e7f3fe69..97227763c00b 100755 --- a/bin/ckmake +++ b/bin/ckmake @@ -1,267 +1,327 @@ -#!/bin/bash -# -# Copyright (C) 2012, Luis R. Rodriguez -# Copyright (C) 2012, Hauke Mehrtens +#!/usr/bin/env python + +# ncurses Linux kernel cross kernel compilation utility + +# Copyright (C) 2012 Luis R. Rodriguez # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -# -# You can use this to compile a module accross all installed kernels -# found. This relies on distribution specific kernels, but can handle -# your own custom list of target kernels. Log is setnt to LOG variable. - - -#export KCFLAGS="-Wno-unused-but-set-variable" -KERNEL_DIR="/lib/modules" -KLIBS="" -LOG="ckmake.log" -LOG_TMP="ckmake-tmp.log" -REPORT="ckmake-report.log" -TIME="0" -DEBUG="0" -NOCOLOR="0" -ARGS="" -QUIET="" -RET_FILE="ret-tmp.txt" - -# First and last kernels to use -FIRST="" -LAST="" -RANGE="" - -RET="" -NUM_CPUS=$(echo $(($(cat /proc/cpuinfo | grep processor | tail -1 | awk '{print $3}')+1))) - -# If $HOME/compat-ksrc is found use that, otherwise use system-wide -# sources in /usr/src. -KSRC_PREFIX= -if [[ -d "$HOME/compat-ksrc" ]]; then - KSRC_PREFIX="$HOME/compat-ksrc" -fi - -# Colorify output if NOCOLOR != 1 or -n is not given -function prettify() -{ - if [[ $NOCOLOR == "1" ]]; then - echo -n "$2" - else - ANSI_CODE= - NORMAL="\033[00m" - case $1 in - "green") - ANSI_CODE="\033[01;32m" - ;; - "yellow") - ANSI_CODE="\033[01;33m" - ;; - "blue") - ANSI_CODE="\033[34m" - ;; - "red") - ANSI_CODE="\033[31m" - ;; - "purple") - ANSI_CODE="\033[35m" - ;; - "cyan") - ANSI_CODE="\033[36m" - ;; - "underline") - ANSI_CODE="\033[02m" - ;; - esac - - echo -e -n "${ANSI_CODE}$2${NORMAL}" - fi -} -function tee_color_split() -{ - while read; do - echo -e $REPLY | perl -pe 's|(\e)\[(\d+)(;*)(\d*)(\w)||g' >> $1 - echo -e $REPLY - done -} - -function log_try_kernel() -{ - printf "Trying kernel %40s\t" "$(prettify blue $1)" -} - -function usage() -{ - echo -e "Usage: $0 [-t] " - echo -e "-t will run: 'time ckmake; time ckmake' account for" - echo -e " benchmark how long it takes to compile without ccache" - echo -e " and a run after cache kicks in" - echo -e "-n Do not use colors in the output" - echo -e "-q will ask ckmake to run make with -s to only output errors" - echo - echo -e " is the arguments you want to pass to the" - echo -e "child make call that ckmake will use. For example if you have" - echo -e "a target 'linux' on your Makefile you can run 'cmake linux'" - echo -e "" - echo -e " are the kernels you want to test" - echo -e "compile against. This consists of a range. The third extraversion" - echo -e "number is ignored" -} - -for i in $@ ; do - case $1 in - "-h") - usage - exit 1 - ;; - "--help") - usage - exit 1 - ;; - "-t") - TIME="1" - shift - ;; - "-n") - NOCOLOR="1" - shift - ;; - "-s") - QUIET="-s" - shift - ;; - "-d") - DEBUG="1" - shift - ;; - *) - echo $i | grep "\.\." 2>&1 > /dev/null - if [[ $? -eq 0 ]]; then - FIRST=$(echo $i | sed 's|\.\.|-|' | awk -F"-" '{print $1}') - LAST=$(echo $i | sed 's|\.\.|-|' | awk -F"-" '{print $2}') - RANGE="${FIRST}..${LAST}" - echo -e "Going to use kernel ranges: $(prettify blue $FIRST)..$(prettify blue $LAST)" - shift - fi - - ARGS="${ARGS} $1" - shift - esac -done - -function run_ckmake() -{ - for i in $KLIBS; do - KERNEL=$(basename $i) - DIR=${i}/build/ - echo -e "--------------------------------------------" >> $LOG - - if [[ ! -d $DIR ]]; then +import locale +import curses +import time +import os +import re +import random +import tempfile +import subprocess +import sys +import signal + +from Queue import * +from threading import Thread, Lock +from shutil import copytree, ignore_patterns, rmtree, copyfileobj + +releases_processed = [] +releases_baking = [] +processed_lock = Lock() +baking_lock = Lock() +my_cwd = os.getcwd() +ckmake_return = 0 + +tmp_path = my_cwd + "/.tmp.ckmake" +ckmake_log = my_cwd + "/ckmake.log" +ckmake_report = my_cwd + "/ckmake-report.log" + +home = os.getenv("HOME") +ksrc = home + "/" + "compat-ksrc" +modules = ksrc + "/lib/modules" + +def clean(): + if os.path.exists(tmp_path): + rmtree(tmp_path) + +def get_processed_rel(i): + "Because... list.pop(i) didn't work to keep order..." + processed_lock.acquire() + for rel in releases_processed: + if (rel['idx'] == i): + processed_lock.release() + return rel + processed_lock.release() + + baking_lock.acquire() + for rel in releases_baking: + if (rel['idx'] == i): + releases_baking.remove(rel) + baking_lock.release() + return rel + baking_lock.release() + +def get_status_name(status): + if (status == 0): + return 'OK' + if (status == 2): + return 'OK' + elif (status == 130): + return 'TERM' + elif (status == 1234): + return 'INIT' + elif (status == -2): + return 'TERM' + else: + return 'FAIL' + +def get_stat_pos(status): + if (status == 0): + return 34 + if (status == 2): + return 34 + elif (status == 130): + return 33 + elif (status == 1234): + return 33 + elif (status == -2): + return 33 + else: + return 33 + +def get_status_color(status): + if (status == 0): + return curses.color_pair(1) + if (status == 2): + return curses.color_pair(1) + elif (status == 130): + return curses.color_pair(2) + elif (status == 1234): + return curses.color_pair(2) + elif (status == -2): + return curses.color_pair(2) + else: + return curses.color_pair(3) + +def print_report(): + os.system("clear") + sys.stdout.write("\n\n\n") + sys.stdout.write("== ckmake-report.log ==\n\n") + report = open(ckmake_report, 'r+') + copyfileobj(report, sys.stdout) + report.close() + +def process_logs(): + os.system('clear') + log = open(ckmake_log, 'w+') + report = open(ckmake_report, 'w+') + for i in range(0, len(releases_processed)): + rel = get_processed_rel(i) + rel_log = open(rel['log'], 'r+') + log.write(rel_log.read()) + status = get_status_name(rel['status']) + rel_report = "%-4s%-20s[ %s ]\n" % \ + ( rel['idx']+1, rel['version'], status) + report.write(rel_report) + if (rel['status'] != 0 and + rel['status'] != 2): + ckmake_return = -1 + report.close() + log.close() + print_report() + +def process_kernel(num, kset): + while True: + rel = kset.queue.get() + work_dir = tmp_path + '/' + rel['version'] + copytree(my_cwd, \ + work_dir, \ + ignore=ignore_patterns('\.git*', '.tmp*', ".git")) + build = '%s/build/' % rel['full_path'] + jobs = ' -j%d ' % kset.build_jobs + make_args = 'KLIB=%(build)s KLIB_BUILD=%(build)s' % \ + { "build": build } + log_file = work_dir + '/' + 'ckmake.n.log' + rel['log'] = log_file + # XXX: figure out how to properly address logging + # without this nasty ass hack. + log = ' > %(log)s 2>&1' % { "log": log_file } + cmd = 'make ' + jobs + make_args + log + + kset.baking(rel) + + p = subprocess.Popen([cmd], + cwd = work_dir, + stdout=None, + stderr=None, + shell=True) + p.wait() + + kset.update_status(rel, p.returncode) + kset.queue.task_done() + kset.completed(rel) + +def cpu_info_build_jobs(): + if not os.path.exists('/proc/cpuinfo'): + return 1 + f = open('/proc/cpuinfo', 'r') + max_cpus = 1 + for line in f: + m = re.match(r"(?Pprocessor\s*:)\s*" \ + "(?P\d+)", + line) + if not m: continue - fi - - NOCOLOR="1" log_try_kernel $KERNEL >> $LOG - log_try_kernel $KERNEL - - #ionice -c 3 nice -n 20 make $QUIET KLIB=$DIR KLIB_BUILD=$DIR -j${NUM_CPUS} -Wunused-but-set-variable $ARGS &>> $LOG - ionice -c 3 nice -n 20 make $QUIET KLIB=$DIR KLIB_BUILD=$DIR -j${NUM_CPUS} $ARGS &>> $LOG - CUR_RET=$? - - if [[ $RET = "" ]]; then - RET=$CUR_RET - fi - - if [[ $CUR_RET -eq 0 ]]; then - echo -e "$(prettify green [OK])" | tee_color_split $LOG - else - echo -e "$(prettify red [FAILED])" | tee_color_split $LOG - RET=$CUR_RET - fi - - nice make clean KLIB=$DIR KLIB_BUILD=$DIR 2>&1 >> $LOG - done - # Bash doesn't do what we expect with the variables... - # and using return $RET here won't work here either given - # that we end up piping the result anyway. - echo $RET > $RET_FILE -} - -# This mimic's the kernel's own algorithm: -# -# KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c)) -function kernel_version_orig { - echo "$@" | awk -F. '{ printf("%d\n", lshift($1,16) + lshift($2, 8) + $3); }' -} - -function kernel_version_26 { - kernel_version_orig $@ -} - -# Ignores the last extraversion number -function kernel_version_30 { - echo "$@" | awk -F. '{ printf("%d\n", lshift($1,16) + lshift($2, 8) ); }' -} - -# If we're using 3.0 kernels we do not require an extraversion, -# although one could be supplied. For purposes of this script -# though the 2.6.29..3.1 ranges are acceptable. If we forced usage -# of kernel_version_orig() for 3.0 it means we'd have to require a user -# to be very specific and specific 2.6.29..3.1.0 or whatever. Lets -# instead be flexible. -function kernel_version { - if [[ $(kernel_version_orig $@ ) -lt $(kernel_version_orig "3.0") ]] ; then - echo $(kernel_version_26 $@) - else - echo $(kernel_version_30 $@) - fi -} - -for i in $(find $KSRC_PREFIX/lib/modules/ -type d -name \*generic\* | sort -n -r | grep -v -E '\-[[:alnum:]]{1,2}\-'); do - KERNEL=$(echo ${i} | awk -F"/" '{print $NF}' | awk -F"-" '{print $1}') - - if [[ ! -z $FIRST && ! -z $LAST ]]; then - if [[ $(kernel_version $KERNEL ) -lt $(kernel_version $FIRST) ]] ; then - continue; - fi - - if [[ $(kernel_version $KERNEL ) -gt $(kernel_version $LAST) ]] ; then - continue; - fi - - if [[ ! -z $DEBUG ]]; then - echo -e "$(prettify cyan $FIRST) $(kernel_version $FIRST) <= $(prettify green $KERNEL) $(kernel_version $KERNEL) <= $(prettify cyan $LAST) $(kernel_version $LAST)" - fi - fi - - KLIBS="$KLIBS $i" -done - -for i in $LOG $LOG_TMP $REPORT; do - echo > $i -done - -DIR="$(echo $KLIBS | awk '{print $1}')/build" -nice make clean KLIB=$DIR KLIB_BUILD=$DIR 2>&1 >> $LOG - -if [[ $TIME != "1" ]]; then - run_ckmake | tee_color_split $REPORT - - cat $LOG $REPORT > $LOG_TMP - mv $LOG_TMP $LOG - rm -f $LOG_TMP - - RET=$(cat $RET_FILE) - exit $RET -fi - -time $0 $QUIET ${RANGE} $ARGS | tee_color_split $REPORT -time $0 $QUIET ${RANGE} $ARGS | egrep "real|user|sys" | tee_color_split $REPORT - -cat $LOG $REPORT > $LOG_TMP -mv $LOG_TMP $LOG - -rm -f $LOG_TMP - -RET=$(cat $RET_FILE) -rm -f $RET_FILE - -exit $RET + proc_specs = m.groupdict() + if (proc_specs['NUM'] > max_cpus): + max_cpus = proc_specs['NUM'] + return int(max_cpus) + 1 + +def kill_curses(): + curses.endwin() + sys.stdout = os.fdopen(0, 'w', 0) + +def sig_handler(signal, frame): + kill_curses() + process_logs() + clean() + sys.exit(-2) + +class kernel_set(): + def __init__(self, stdscr): + self.queue = Queue() + self.releases = [] + self.stdscr = stdscr + self.lock = Lock() + signal.signal(signal.SIGINT, sig_handler) + self.build_jobs = cpu_info_build_jobs() + if (curses.has_colors()): + curses.init_pair(1, + curses.COLOR_GREEN, + curses.COLOR_BLACK) + curses.init_pair(2, + curses.COLOR_CYAN, + curses.COLOR_BLACK) + curses.init_pair(3, + curses.COLOR_RED, + curses.COLOR_BLACK) + curses.init_pair(4, + curses.COLOR_BLUE, + curses.COLOR_BLACK) + def baking(self, rel): + baking_lock.acquire() + releases_baking.append(rel) + baking_lock.release() + def completed(self, rel): + processed_lock.acquire() + releases_processed.insert(rel['idx'], rel) + processed_lock.release() + def set_locale(self): + locale.setlocale(locale.LC_ALL, '') + code = locale.getpreferredencoding() + def refresh(self): + self.lock.acquire() + self.stdscr.refresh() + self.lock.release() + def parse_releases(self): + for dirname, dirnames, filenames in os.walk(modules): + dirnames.sort() + for subdirname in dirnames: + m = re.match(r"v*(?P\w+.)" \ + "(?P\w+.*)" \ + "(?P\w*)" \ + "(?P[.-]\w*)" \ + "(?P[-]\w*).*", \ + subdirname) + if not m: + continue + + specifics = m.groupdict() + + ver = specifics['VERSION'] + \ + specifics['PATCHLEVEL'] + \ + specifics['SUBLEVEL'] + + for rel in self.releases: + if (rel['version'] == ver): + continue + + rel = dict(idx=len(self.releases), + name=subdirname, + full_path=dirname + '/' + + subdirname, + version=ver, + processed=False, + log='', + status=1234) + self.releases.insert(rel['idx'], rel) + self.refresh() + def setup_screen(self): + for i in range(0, len(self.releases)): + rel = self.releases[i] + self.lock.acquire() + self.stdscr.addstr(rel['idx'], 0, + "%-4d" % (rel['idx']+1)) + if (curses.has_colors()): + self.stdscr.addstr(rel['idx'], 5, + "%-20s" % (rel['version']), + curses.color_pair(4)) + else: + self.stdscr.addstr(rel['idx'], 0, + "%-20s" % (rel['version'])) + self.stdscr.addstr(rel['idx'], 30, "[ ]") + self.stdscr.refresh() + self.lock.release() + def create_threads(self): + for rel in self.releases: + th = Thread(target=process_kernel, args=(0, self)) + th.setDaemon(True) + th.start() + def kick_threads(self): + for rel in self.releases: + self.queue.put(rel) + def wait_threads(self): + self.queue.join() + def update_status(self, rel, status): + self.lock.acquire() + stat_name = get_status_name(status) + stat_pos = get_stat_pos(status) + if (curses.has_colors()): + stat_color = get_status_color(status) + self.stdscr.addstr(rel['idx'], stat_pos, stat_name, + get_status_color(status)) + else: + self.stdscr.addstr(rel['idx'], stat_pos, stat_name) + rel['processed'] = True + rel['status'] = status + self.stdscr.refresh() + self.lock.release() + +def main(stdscr): + kset = kernel_set(stdscr) + + kset.set_locale() + kset.parse_releases() + kset.setup_screen() + kset.create_threads() + kset.kick_threads() + kset.wait_threads() + kset.refresh() + +if __name__ == "__main__": + if not os.path.exists(modules): + print "%s does not exist" % (modules) + sys.exit(1) + if not os.path.exists(my_cwd + '/Makefile'): + print "%s does not exist" % (my_cwd + '/Makefile') + sys.exit(1) + if os.path.exists(ckmake_log): + os.remove(ckmake_log) + if os.path.exists(ckmake_report): + os.remove(ckmake_report) + if os.path.exists(tmp_path): + rmtree(tmp_path) + os.makedirs(tmp_path) + curses.wrapper(main) + kill_curses() + process_logs() + clean() + sys.exit(ckmake_return) -- 2.30.2