-#!/bin/bash
-#
-# Copyright (C) 2012, Luis R. Rodriguez <mcgrof@frijolero.org>
-# Copyright (C) 2012, Hauke Mehrtens <hauke@hauke-m.de>
+#!/usr/bin/env python
+
+# ncurses Linux kernel cross kernel compilation utility
+
+# Copyright (C) 2012 Luis R. Rodriguez <mcgrof@do-not-panic.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
-#
-# You can use this to compile a module accross all installed kernels
-# found. This relies on distribution specific kernels, but can handle
-# your own custom list of target kernels. Log is setnt to LOG variable.
-
-
-#export KCFLAGS="-Wno-unused-but-set-variable"
-KERNEL_DIR="/lib/modules"
-KLIBS=""
-LOG="ckmake.log"
-LOG_TMP="ckmake-tmp.log"
-REPORT="ckmake-report.log"
-TIME="0"
-DEBUG="0"
-NOCOLOR="0"
-ARGS=""
-QUIET=""
-RET_FILE="ret-tmp.txt"
-
-# First and last kernels to use
-FIRST=""
-LAST=""
-RANGE=""
-
-RET=""
-NUM_CPUS=$(echo $(($(cat /proc/cpuinfo | grep processor | tail -1 | awk '{print $3}')+1)))
-
-# If $HOME/compat-ksrc is found use that, otherwise use system-wide
-# sources in /usr/src.
-KSRC_PREFIX=
-if [[ -d "$HOME/compat-ksrc" ]]; then
- KSRC_PREFIX="$HOME/compat-ksrc"
-fi
-
-# Colorify output if NOCOLOR != 1 or -n is not given
-function prettify()
-{
- if [[ $NOCOLOR == "1" ]]; then
- echo -n "$2"
- else
- ANSI_CODE=
- NORMAL="\033[00m"
- case $1 in
- "green")
- ANSI_CODE="\033[01;32m"
- ;;
- "yellow")
- ANSI_CODE="\033[01;33m"
- ;;
- "blue")
- ANSI_CODE="\033[34m"
- ;;
- "red")
- ANSI_CODE="\033[31m"
- ;;
- "purple")
- ANSI_CODE="\033[35m"
- ;;
- "cyan")
- ANSI_CODE="\033[36m"
- ;;
- "underline")
- ANSI_CODE="\033[02m"
- ;;
- esac
-
- echo -e -n "${ANSI_CODE}$2${NORMAL}"
- fi
-}
-function tee_color_split()
-{
- while read; do
- echo -e $REPLY | perl -pe 's|(\e)\[(\d+)(;*)(\d*)(\w)||g' >> $1
- echo -e $REPLY
- done
-}
-
-function log_try_kernel()
-{
- printf "Trying kernel %40s\t" "$(prettify blue $1)"
-}
-
-function usage()
-{
- echo -e "Usage: $0 [-t] <optional-target> <first_kernel..last_kernel>"
- echo -e "-t will run: 'time ckmake; time ckmake' account for"
- echo -e " benchmark how long it takes to compile without ccache"
- echo -e " and a run after cache kicks in"
- echo -e "-n Do not use colors in the output"
- echo -e "-q will ask ckmake to run make with -s to only output errors"
- echo
- echo -e "<optional-target> is the arguments you want to pass to the"
- echo -e "child make call that ckmake will use. For example if you have"
- echo -e "a target 'linux' on your Makefile you can run 'cmake linux'"
- echo -e ""
- echo -e "<first_kernel..last_kernel> are the kernels you want to test"
- echo -e "compile against. This consists of a range. The third extraversion"
- echo -e "number is ignored"
-}
-
-for i in $@ ; do
- case $1 in
- "-h")
- usage
- exit 1
- ;;
- "--help")
- usage
- exit 1
- ;;
- "-t")
- TIME="1"
- shift
- ;;
- "-n")
- NOCOLOR="1"
- shift
- ;;
- "-s")
- QUIET="-s"
- shift
- ;;
- "-d")
- DEBUG="1"
- shift
- ;;
- *)
- echo $i | grep "\.\." 2>&1 > /dev/null
- if [[ $? -eq 0 ]]; then
- FIRST=$(echo $i | sed 's|\.\.|-|' | awk -F"-" '{print $1}')
- LAST=$(echo $i | sed 's|\.\.|-|' | awk -F"-" '{print $2}')
- RANGE="${FIRST}..${LAST}"
- echo -e "Going to use kernel ranges: $(prettify blue $FIRST)..$(prettify blue $LAST)"
- shift
- fi
-
- ARGS="${ARGS} $1"
- shift
- esac
-done
-
-function run_ckmake()
-{
- for i in $KLIBS; do
- KERNEL=$(basename $i)
- DIR=${i}/build/
- echo -e "--------------------------------------------" >> $LOG
-
- if [[ ! -d $DIR ]]; then
+import locale
+import curses
+import time
+import os
+import re
+import random
+import tempfile
+import subprocess
+import sys
+import signal
+
+from Queue import *
+from threading import Thread, Lock
+from shutil import copytree, ignore_patterns, rmtree, copyfileobj
+
+releases_processed = []
+releases_baking = []
+processed_lock = Lock()
+baking_lock = Lock()
+my_cwd = os.getcwd()
+ckmake_return = 0
+
+tmp_path = my_cwd + "/.tmp.ckmake"
+ckmake_log = my_cwd + "/ckmake.log"
+ckmake_report = my_cwd + "/ckmake-report.log"
+
+home = os.getenv("HOME")
+ksrc = home + "/" + "compat-ksrc"
+modules = ksrc + "/lib/modules"
+
+def clean():
+ if os.path.exists(tmp_path):
+ rmtree(tmp_path)
+
+def get_processed_rel(i):
+ "Because... list.pop(i) didn't work to keep order..."
+ processed_lock.acquire()
+ for rel in releases_processed:
+ if (rel['idx'] == i):
+ processed_lock.release()
+ return rel
+ processed_lock.release()
+
+ baking_lock.acquire()
+ for rel in releases_baking:
+ if (rel['idx'] == i):
+ releases_baking.remove(rel)
+ baking_lock.release()
+ return rel
+ baking_lock.release()
+
+def get_status_name(status):
+ if (status == 0):
+ return 'OK'
+ if (status == 2):
+ return 'OK'
+ elif (status == 130):
+ return 'TERM'
+ elif (status == 1234):
+ return 'INIT'
+ elif (status == -2):
+ return 'TERM'
+ else:
+ return 'FAIL'
+
+def get_stat_pos(status):
+ if (status == 0):
+ return 34
+ if (status == 2):
+ return 34
+ elif (status == 130):
+ return 33
+ elif (status == 1234):
+ return 33
+ elif (status == -2):
+ return 33
+ else:
+ return 33
+
+def get_status_color(status):
+ if (status == 0):
+ return curses.color_pair(1)
+ if (status == 2):
+ return curses.color_pair(1)
+ elif (status == 130):
+ return curses.color_pair(2)
+ elif (status == 1234):
+ return curses.color_pair(2)
+ elif (status == -2):
+ return curses.color_pair(2)
+ else:
+ return curses.color_pair(3)
+
+def print_report():
+ os.system("clear")
+ sys.stdout.write("\n\n\n")
+ sys.stdout.write("== ckmake-report.log ==\n\n")
+ report = open(ckmake_report, 'r+')
+ copyfileobj(report, sys.stdout)
+ report.close()
+
+def process_logs():
+ os.system('clear')
+ log = open(ckmake_log, 'w+')
+ report = open(ckmake_report, 'w+')
+ for i in range(0, len(releases_processed)):
+ rel = get_processed_rel(i)
+ rel_log = open(rel['log'], 'r+')
+ log.write(rel_log.read())
+ status = get_status_name(rel['status'])
+ rel_report = "%-4s%-20s[ %s ]\n" % \
+ ( rel['idx']+1, rel['version'], status)
+ report.write(rel_report)
+ if (rel['status'] != 0 and
+ rel['status'] != 2):
+ ckmake_return = -1
+ report.close()
+ log.close()
+ print_report()
+
+def process_kernel(num, kset):
+ while True:
+ rel = kset.queue.get()
+ work_dir = tmp_path + '/' + rel['version']
+ copytree(my_cwd, \
+ work_dir, \
+ ignore=ignore_patterns('\.git*', '.tmp*', ".git"))
+ build = '%s/build/' % rel['full_path']
+ jobs = ' -j%d ' % kset.build_jobs
+ make_args = 'KLIB=%(build)s KLIB_BUILD=%(build)s' % \
+ { "build": build }
+ log_file = work_dir + '/' + 'ckmake.n.log'
+ rel['log'] = log_file
+ # XXX: figure out how to properly address logging
+ # without this nasty ass hack.
+ log = ' > %(log)s 2>&1' % { "log": log_file }
+ cmd = 'make ' + jobs + make_args + log
+
+ kset.baking(rel)
+
+ p = subprocess.Popen([cmd],
+ cwd = work_dir,
+ stdout=None,
+ stderr=None,
+ shell=True)
+ p.wait()
+
+ kset.update_status(rel, p.returncode)
+ kset.queue.task_done()
+ kset.completed(rel)
+
+def cpu_info_build_jobs():
+ if not os.path.exists('/proc/cpuinfo'):
+ return 1
+ f = open('/proc/cpuinfo', 'r')
+ max_cpus = 1
+ for line in f:
+ m = re.match(r"(?P<PROC>processor\s*:)\s*" \
+ "(?P<NUM>\d+)",
+ line)
+ if not m:
continue
- fi
-
- NOCOLOR="1" log_try_kernel $KERNEL >> $LOG
- log_try_kernel $KERNEL
-
- #ionice -c 3 nice -n 20 make $QUIET KLIB=$DIR KLIB_BUILD=$DIR -j${NUM_CPUS} -Wunused-but-set-variable $ARGS &>> $LOG
- ionice -c 3 nice -n 20 make $QUIET KLIB=$DIR KLIB_BUILD=$DIR -j${NUM_CPUS} $ARGS &>> $LOG
- CUR_RET=$?
-
- if [[ $RET = "" ]]; then
- RET=$CUR_RET
- fi
-
- if [[ $CUR_RET -eq 0 ]]; then
- echo -e "$(prettify green [OK])" | tee_color_split $LOG
- else
- echo -e "$(prettify red [FAILED])" | tee_color_split $LOG
- RET=$CUR_RET
- fi
-
- nice make clean KLIB=$DIR KLIB_BUILD=$DIR 2>&1 >> $LOG
- done
- # Bash doesn't do what we expect with the variables...
- # and using return $RET here won't work here either given
- # that we end up piping the result anyway.
- echo $RET > $RET_FILE
-}
-
-# This mimic's the kernel's own algorithm:
-#
-# KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))
-function kernel_version_orig {
- echo "$@" | awk -F. '{ printf("%d\n", lshift($1,16) + lshift($2, 8) + $3); }'
-}
-
-function kernel_version_26 {
- kernel_version_orig $@
-}
-
-# Ignores the last extraversion number
-function kernel_version_30 {
- echo "$@" | awk -F. '{ printf("%d\n", lshift($1,16) + lshift($2, 8) ); }'
-}
-
-# If we're using 3.0 kernels we do not require an extraversion,
-# although one could be supplied. For purposes of this script
-# though the 2.6.29..3.1 ranges are acceptable. If we forced usage
-# of kernel_version_orig() for 3.0 it means we'd have to require a user
-# to be very specific and specific 2.6.29..3.1.0 or whatever. Lets
-# instead be flexible.
-function kernel_version {
- if [[ $(kernel_version_orig $@ ) -lt $(kernel_version_orig "3.0") ]] ; then
- echo $(kernel_version_26 $@)
- else
- echo $(kernel_version_30 $@)
- fi
-}
-
-for i in $(find $KSRC_PREFIX/lib/modules/ -type d -name \*generic\* | sort -n -r | grep -v -E '\-[[:alnum:]]{1,2}\-'); do
- KERNEL=$(echo ${i} | awk -F"/" '{print $NF}' | awk -F"-" '{print $1}')
-
- if [[ ! -z $FIRST && ! -z $LAST ]]; then
- if [[ $(kernel_version $KERNEL ) -lt $(kernel_version $FIRST) ]] ; then
- continue;
- fi
-
- if [[ $(kernel_version $KERNEL ) -gt $(kernel_version $LAST) ]] ; then
- continue;
- fi
-
- if [[ ! -z $DEBUG ]]; then
- echo -e "$(prettify cyan $FIRST) $(kernel_version $FIRST) <= $(prettify green $KERNEL) $(kernel_version $KERNEL) <= $(prettify cyan $LAST) $(kernel_version $LAST)"
- fi
- fi
-
- KLIBS="$KLIBS $i"
-done
-
-for i in $LOG $LOG_TMP $REPORT; do
- echo > $i
-done
-
-DIR="$(echo $KLIBS | awk '{print $1}')/build"
-nice make clean KLIB=$DIR KLIB_BUILD=$DIR 2>&1 >> $LOG
-
-if [[ $TIME != "1" ]]; then
- run_ckmake | tee_color_split $REPORT
-
- cat $LOG $REPORT > $LOG_TMP
- mv $LOG_TMP $LOG
- rm -f $LOG_TMP
-
- RET=$(cat $RET_FILE)
- exit $RET
-fi
-
-time $0 $QUIET ${RANGE} $ARGS | tee_color_split $REPORT
-time $0 $QUIET ${RANGE} $ARGS | egrep "real|user|sys" | tee_color_split $REPORT
-
-cat $LOG $REPORT > $LOG_TMP
-mv $LOG_TMP $LOG
-
-rm -f $LOG_TMP
-
-RET=$(cat $RET_FILE)
-rm -f $RET_FILE
-
-exit $RET
+ proc_specs = m.groupdict()
+ if (proc_specs['NUM'] > max_cpus):
+ max_cpus = proc_specs['NUM']
+ return int(max_cpus) + 1
+
+def kill_curses():
+ curses.endwin()
+ sys.stdout = os.fdopen(0, 'w', 0)
+
+def sig_handler(signal, frame):
+ kill_curses()
+ process_logs()
+ clean()
+ sys.exit(-2)
+
+class kernel_set():
+ def __init__(self, stdscr):
+ self.queue = Queue()
+ self.releases = []
+ self.stdscr = stdscr
+ self.lock = Lock()
+ signal.signal(signal.SIGINT, sig_handler)
+ self.build_jobs = cpu_info_build_jobs()
+ if (curses.has_colors()):
+ curses.init_pair(1,
+ curses.COLOR_GREEN,
+ curses.COLOR_BLACK)
+ curses.init_pair(2,
+ curses.COLOR_CYAN,
+ curses.COLOR_BLACK)
+ curses.init_pair(3,
+ curses.COLOR_RED,
+ curses.COLOR_BLACK)
+ curses.init_pair(4,
+ curses.COLOR_BLUE,
+ curses.COLOR_BLACK)
+ def baking(self, rel):
+ baking_lock.acquire()
+ releases_baking.append(rel)
+ baking_lock.release()
+ def completed(self, rel):
+ processed_lock.acquire()
+ releases_processed.insert(rel['idx'], rel)
+ processed_lock.release()
+ def set_locale(self):
+ locale.setlocale(locale.LC_ALL, '')
+ code = locale.getpreferredencoding()
+ def refresh(self):
+ self.lock.acquire()
+ self.stdscr.refresh()
+ self.lock.release()
+ def parse_releases(self):
+ for dirname, dirnames, filenames in os.walk(modules):
+ dirnames.sort()
+ for subdirname in dirnames:
+ m = re.match(r"v*(?P<VERSION>\w+.)" \
+ "(?P<PATCHLEVEL>\w+.*)" \
+ "(?P<SUBLEVEL>\w*)" \
+ "(?P<EXTRAVERSION>[.-]\w*)" \
+ "(?P<RELMOD>[-]\w*).*", \
+ subdirname)
+ if not m:
+ continue
+
+ specifics = m.groupdict()
+
+ ver = specifics['VERSION'] + \
+ specifics['PATCHLEVEL'] + \
+ specifics['SUBLEVEL']
+
+ for rel in self.releases:
+ if (rel['version'] == ver):
+ continue
+
+ rel = dict(idx=len(self.releases),
+ name=subdirname,
+ full_path=dirname + '/' +
+ subdirname,
+ version=ver,
+ processed=False,
+ log='',
+ status=1234)
+ self.releases.insert(rel['idx'], rel)
+ self.refresh()
+ def setup_screen(self):
+ for i in range(0, len(self.releases)):
+ rel = self.releases[i]
+ self.lock.acquire()
+ self.stdscr.addstr(rel['idx'], 0,
+ "%-4d" % (rel['idx']+1))
+ if (curses.has_colors()):
+ self.stdscr.addstr(rel['idx'], 5,
+ "%-20s" % (rel['version']),
+ curses.color_pair(4))
+ else:
+ self.stdscr.addstr(rel['idx'], 0,
+ "%-20s" % (rel['version']))
+ self.stdscr.addstr(rel['idx'], 30, "[ ]")
+ self.stdscr.refresh()
+ self.lock.release()
+ def create_threads(self):
+ for rel in self.releases:
+ th = Thread(target=process_kernel, args=(0, self))
+ th.setDaemon(True)
+ th.start()
+ def kick_threads(self):
+ for rel in self.releases:
+ self.queue.put(rel)
+ def wait_threads(self):
+ self.queue.join()
+ def update_status(self, rel, status):
+ self.lock.acquire()
+ stat_name = get_status_name(status)
+ stat_pos = get_stat_pos(status)
+ if (curses.has_colors()):
+ stat_color = get_status_color(status)
+ self.stdscr.addstr(rel['idx'], stat_pos, stat_name,
+ get_status_color(status))
+ else:
+ self.stdscr.addstr(rel['idx'], stat_pos, stat_name)
+ rel['processed'] = True
+ rel['status'] = status
+ self.stdscr.refresh()
+ self.lock.release()
+
+def main(stdscr):
+ kset = kernel_set(stdscr)
+
+ kset.set_locale()
+ kset.parse_releases()
+ kset.setup_screen()
+ kset.create_threads()
+ kset.kick_threads()
+ kset.wait_threads()
+ kset.refresh()
+
+if __name__ == "__main__":
+ if not os.path.exists(modules):
+ print "%s does not exist" % (modules)
+ sys.exit(1)
+ if not os.path.exists(my_cwd + '/Makefile'):
+ print "%s does not exist" % (my_cwd + '/Makefile')
+ sys.exit(1)
+ if os.path.exists(ckmake_log):
+ os.remove(ckmake_log)
+ if os.path.exists(ckmake_report):
+ os.remove(ckmake_report)
+ if os.path.exists(tmp_path):
+ rmtree(tmp_path)
+ os.makedirs(tmp_path)
+ curses.wrapper(main)
+ kill_curses()
+ process_logs()
+ clean()
+ sys.exit(ckmake_return)
projects / openwrt / staging / blogic.git / commitdiff