mir/kea
2
0
Fork 0
mirror of https://gitlab.isc.org/isc-projects/kea synced 2025-08-29 04:57:52 +00:00
Code Issues Releases Wiki Activity

hammer: fixed conflict when hammer is used from 2 repo folders on the same computer, added jobs switch for forcing number of parallel compilation processes

Browse Source
This commit is contained in:
Michal Nowikowski 2019-01-22 11:59:25 +01:00
parent cded268ebb
commit 54aefa0782
1 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
hammer.py
View File

@ -10,13 +10,14 @@ from __future__ import print_function
import os import os
import sys import sys
import glob import glob
import argparse
import time import time
import platform import json
import subprocess
import logging import logging
import datetime import datetime
import json import platform
import binascii
import argparse
import subprocess
import multiprocessing import multiprocessing
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
@ -210,11 +211,6 @@ class VagrantEnv(object):
image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant] image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant]
self.repo_dir = os.getcwd() self.repo_dir = os.getcwd()
self.name = "hmr-%s-%s-kea-srv" % (system, sys_revision.replace('.', '-'))
vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
name=self.name)
sys_dir = "%s-%s" % (system, sys_revision) sys_dir = "%s-%s" % (system, sys_revision)
if provider == "virtualbox": if provider == "virtualbox":
self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox') self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
@ -233,9 +229,17 @@ class VagrantEnv(object):
# TODO: destroy any existing VM # TODO: destroy any existing VM
pass pass
crc = binascii.crc32(self.vagrant_dir.encode())
self.name = "hmr-%s-%s-kea-srv-%08d" % (system, sys_revision.replace('.', '-'), crc)
vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
name=self.name)
with open(vagrantfile_path, "w") as f: with open(vagrantfile_path, "w") as f:
f.write(vagrantfile) f.write(vagrantfile)
log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)
def up(self): def up(self):
execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run) execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run) execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
@ -272,7 +276,7 @@ class VagrantEnv(object):
execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir) execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
execute('sudo rm -rf %s' % lxc_box_dir) execute('sudo rm -rf %s' % lxc_box_dir)
def run_build_and_test(self, tarball_path): def run_build_and_test(self, tarball_path, jobs):
if self.dry_run: if self.dry_run:
return 0, 0 return 0, 0
@ -290,7 +294,7 @@ class VagrantEnv(object):
t0 = time.time() t0 = time.time()
bld_cmd = "%s hammer.py build -p local -t %s.tar.gz" % (self.python, name_ver) bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
if self.features_arg: if self.features_arg:
bld_cmd += ' ' + self.features_arg bld_cmd += ' ' + self.features_arg
if self.nofeatures_arg: if self.nofeatures_arg:
@ -689,7 +693,7 @@ def prepare_deps_local(features, check_times):
#execute('sudo rm -rf /usr/share/doc') #execute('sudo rm -rf /usr/share/doc')
def build_local(features, tarball_path, check_times): def build_local(features, tarball_path, check_times, jobs):
env = os.environ.copy() env = os.environ.copy()
env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C' env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
@ -790,11 +794,14 @@ def build_local(features, tarball_path, check_times):
execute(cmd, cwd=src_path, env=env, check_times=check_times) execute(cmd, cwd=src_path, env=env, check_times=check_times)
cpus = multiprocessing.cpu_count() - 1 if jobs == 0:
if distro == 'centos': cpus = multiprocessing.cpu_count() - 1
cpus = cpus // 2 if distro == 'centos':
if cpus == 0: cpus = cpus // 2
cpus = 1 if cpus == 0:
cpus = 1
else:
cpus = jobs
cmd = 'make -j%s' % cpus cmd = 'make -j%s' % cpus
execute(cmd, cwd=src_path, env=env, timeout=40 * 60, check_times=check_times) execute(cmd, cwd=src_path, env=env, timeout=40 * 60, check_times=check_times)
@ -844,7 +851,7 @@ def build_local(features, tarball_path, check_times):
execute('df -h') execute('df -h')
def build_in_vagrant(provider, system, sys_revision, features, leave_system, tarball_path, dry_run, quiet, clean_start, check_times): def build_in_vagrant(provider, system, sys_revision, features, leave_system, tarball_path, dry_run, quiet, clean_start, check_times, jobs):
log.info('') log.info('')
log.info(">>> Building %s, %s, %s" % (provider, system, sys_revision)) log.info(">>> Building %s, %s, %s" % (provider, system, sys_revision))
log.info('') log.info('')
@ -860,7 +867,7 @@ def build_in_vagrant(provider, system, sys_revision, features, leave_system, tar
ve.destroy(force=True) ve.destroy(force=True)
ve.up() ve.up()
ve.prepare_deps() ve.prepare_deps()
total, passed = ve.run_build_and_test(tarball_path) total, passed = ve.run_build_and_test(tarball_path, jobs)
msg = ' - ' + green('all ok') msg = ' - ' + green('all ok')
except KeyboardInterrupt as e: except KeyboardInterrupt as e:
error = e error = e
@ -962,6 +969,7 @@ def parse_args():
parser.add_argument('-n', '--dry-run', action='store_true', help='Print only what would be done.') parser.add_argument('-n', '--dry-run', action='store_true', help='Print only what would be done.')
parser.add_argument('-c', '--clean-start', action='store_true', help='If there is pre-existing system then it is destroyed first.') parser.add_argument('-c', '--clean-start', action='store_true', help='If there is pre-existing system then it is destroyed first.')
parser.add_argument('-i', '--check-times', action='store_true', help='Do not allow executing commands infinitelly.') parser.add_argument('-i', '--check-times', action='store_true', help='Do not allow executing commands infinitelly.')
parser.add_argument('-j', '--jobs', default=0, help='Number of processes used in compilation. Override make -j default value.')
args = parser.parse_args() args = parser.parse_args()
@ -1049,7 +1057,7 @@ def main():
elif args.command == "build": elif args.command == "build":
log.info('Enabled features: %s', ' '.join(features)) log.info('Enabled features: %s', ' '.join(features))
if args.provider == 'local': if args.provider == 'local':
build_local(features, args.from_tarball, args.check_times) build_local(features, args.from_tarball, args.check_times, int(args.jobs))
return return
if args.provider == 'all': if args.provider == 'all':
@ -1084,7 +1092,7 @@ def main():
fail = False fail = False
for provider, system, revision in plan: for provider, system, revision in plan:
duration, error, total, passed = build_in_vagrant(provider, system, revision, features, args.leave_system, args.from_tarball, duration, error, total, passed = build_in_vagrant(provider, system, revision, features, args.leave_system, args.from_tarball,
args.dry_run, args.quiet, args.clean_start, args.check_times) args.dry_run, args.quiet, args.clean_start, args.check_times, int(args.jobs))
results[(provider, system, revision)] = (duration, error, total, passed) results[(provider, system, revision)] = (duration, error, total, passed)
if error: if error: