#!/usr/bin/env python3 import sys, json, yaml import os import subprocess from shutil import copy2, move from os.path import expanduser import time import datetime from util_functions import util_functions from util_functions import at_functions from util_functions import cdash_functions from util_functions import step_functions # doyAnt arguments: # 1. HOST # 2. DIRN # 3. os of slave # 4. os size of slave # 5. platform of slave # 6. configuration # 7. ctparams # 8. build system # 9. local folder for file # 10. ant target for test # 11. btconfig # 12. datastore def main(argv): print ('{}:{}:{}:{}:{}:{}:{}:{}:{}:{}:{}:{}'.format( sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6],sys.argv[7], sys.argv[8],sys.argv[9],sys.argv[10],sys.argv[11],sys.argv[12])) HOST = sys.argv[1] DIRN = sys.argv[2] slaveos = sys.argv[3] theossize = sys.argv[4] theplatform = sys.argv[5] buildconfig = sys.argv[6] ctparams = sys.argv[7] buildsys = sys.argv[8] installocation = sys.argv[9] testtype = sys.argv[10] configfile = sys.argv[11] datastore = {} datastore = eval(argv[12]) toolsets = datastore['toolsets'] compilers = datastore['compilers'] modules = datastore['modules'] generator = datastore['generator'] generatorarch = datastore['gen_arch'] schedule = datastore['scheduler'] compilername = util_functions.VS_full_to_short(generator, theossize) script_path, script_name = os.path.split(sys.argv[0]) userhome = expanduser("~") bbuserhome = os.path.join(userhome, '.buildbot') util_functions.getPlatforms(buildsys) if theplatform.lower() in util_functions.WinPlatforms: slaveos = 'windows{}-{}'.format(theplatform, theossize) script_res = {ctparams+'-'+testtype+'-'+script_name: 'SKIPPED'} thereturncode = 0 if not 'override' in schedule.lower(): if os.path.exists('{}'.format(bbuserhome)): worker_params_file = os.path.join(bbuserhome, 'bbworker.json') try: with open(worker_params_file, 'r') as json_worker_params_file: json_worker_params_data = json.load(json_worker_params_file) toolsets = json_worker_params_data['worker']['toolsets'] compilers = json_worker_params_data['worker']['compilers'] compilervers = json_worker_params_data['worker']['compilervers'] modules = json_worker_params_data['worker']['modules'] if datastore['gen_arch']: generatorarch = datastore['gen_arch'] #print('compilervers={}'.format(compilervers)) except Exception as e: print('user home bbworker file failure {}'.format(e), flush=True) toolsets = {'default': ['default']} compilers = ['Java'] compilervers = [{"compiler":"Java", "module":"JAVA", "name":"OPENJDK", "command":"java", "version":"1.8"}] modules = {'use': ''} else: toolsets = datastore['toolsets'] compilers = datastore['compilers'] compilervers = [{'compiler':'default', 'module':'default', 'name':'default', 'command':'default', 'version':'default'}] modules = datastore['modules'] subprocreturncode = 255 scriptdir = os.getcwd() #### print('scriptdir={}'.format(scriptdir), flush=True) source_dir = os.path.join(scriptdir, '..', 'hdfsrc') print('source_dir={}'.format(source_dir), flush=True) jsparams_file = 'bb_params.json' if scriptdir.split(os.sep)[-1] == buildsys: jsparams_file = '../bb_params.json' with open(jsparams_file, 'r') as json_bbparams_file: json_bbparams_data = json.load(json_bbparams_file) with open(configfile, 'r') as yaml_file: yaml_data = yaml.safe_load(yaml_file) #print(yaml_data) currdir=os.getcwd() #### thescriptreturncode = 0 if buildsys in yaml_data['configfile'][buildconfig][ctparams]['buildsys']: script_tc_res = {ctparams+'-'+testtype+'-'+script_name+'-config': 'SKIPPED'} target_binaries = [] target_binaries = util_functions.format_binary_name_list(theplatform, yaml_data['configfile'], buildconfig, ctparams, buildsys, compilername, theossize, True, False) print ('target_binaries {}'.format(target_binaries), flush=True) for ctparam in yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]: if 'testparams' in ctparam: testparams = yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]['testparams'] for testparam in testparams: if 'usetoolsets' in testparam: toolsets = testparams['usetoolsets'] print('toolsets={}'.format(toolsets), flush=True) configToolset = 'default' for toolset in toolsets[configToolset]: thereturncode = 0 install_folder = installocation print('toolset={}'.format(toolset), flush=True) testfolder = testtype #testfolder = os.path.join(source_dir, testtype) os.chdir(scriptdir) currdir=os.getcwd() #### print ('current={}'.format(currdir), flush=True) ts_path = 'hdfbld' ts_mod = 'default' ############################################## # determine config ts path ############################################## if toolset != 'default': if theplatform.lower() in util_functions.WinPlatforms: ts_path = (toolset.split(None, 1))[0] else: if 'mingw' or 'aarch' in toolset: ts_path = (toolset.split(None, 1))[0] else: ts_mod = (toolset.split('/', 1))[0] try: ts_path = (toolset.split('/', 1))[1] for mod in modules: k = mod v = modules[k] util_functions.module(k, v) util_functions.module('load', '{}'.format(toolset)) except: print ('no module use env var', flush=True) ts_path = (toolset.split(None, 1))[0] print ('ts_path={}'.format(ts_path), flush=True) ############################################## # create binary install ############################################## if not os.path.exists('{}'.format(ts_path)): os.mkdir(ts_path) os.chdir(ts_path) testfolder = '{}/{}'.format(ts_path, testfolder) if not ctparams == 'bbparams': print('ctparams testfolder={}'.format(testfolder), flush=True) if not os.path.exists('{}'.format(testfolder)): os.mkdir(testfolder) os.chdir(testfolder) testfolder = '{}/{}'.format(testfolder, ctparams) currinstalldir=os.getcwd() #### builddir = os.getcwd() print('builddir={}'.format(builddir), flush=True) print('testfolder={}'.format(testfolder), flush=True) thereturncode = step_functions.ftp_uncompress(json_bbparams_data, HOST, DIRN, theplatform, toolset, ts_path, install_folder, target_binaries) if thereturncode == 0: for etparam in yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]: if 'envparams' in etparam: envparams = yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]['envparams'] #print ('tconf env == {}'.format(envparams)) if 'ftpbinary' in envparams: if envparams['ftpbinary']: if ctparams == 'bbparams': for envparam in envparams: if 'binaryfolder' in envparam: install_folder += '/' + envparams['binaryfolder'] else: install_folder += '/' + ctparams script_tc_res.update({etparam+'-installfolder':install_folder}) json_bbparams_data.append(script_tc_res) currinstalldir=os.getcwd() #### print ('current install={}'.format(currdir), flush=True) thereturncode = step_functions.install_binary(json_bbparams_data, buildsys, theplatform, install_folder, 'install') os.chdir(currinstalldir) #### if thereturncode == 255: os.chdir(scriptdir) json_bbparams_data.append(script_res) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) raise Exception('Unable to get and uncompress file') publish = 'False' publishSource = 'False' for btparam in yaml_data['configfile'][buildconfig][ctparams]: if 'testparams' in btparam: testparams = yaml_data['configfile'][buildconfig][ctparams]['testparams'] #print ('tconf env == {}'.format(testparams)) for testparam in testparams: #print ('tconf tparm = {}'.format(testparam)) if 'publish' in testparam: publish = testparams['publish'] if 'publishsource' in testparam: publishSource = testparams['publishsource'] currdir=os.getcwd() #### #print ('current={}'.format(currdir)) my_env = os.environ.copy() for aparam in yaml_data['configfile'][buildconfig]['bbparams']: if 'testparams' in aparam: testparams = yaml_data['configfile'][buildconfig]['bbparams']['testparams'] for testparam in testparams: if 'env' == testparam.lower(): envitems = testparams['env'] for envitem in envitems: k = envitem v = envitems[k] my_env[str(k)] = str(v) targetnames = [] installpath = '.' for aparam in yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]: if 'testparams' in aparam: testparams = yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]['testparams'] #print ('tconf env == {}'.format(testparams)) for testparam in testparams: #print ('tconf tparm = {}'.format(testparam)) if testtype == testparam.lower(): targetnames = testparams[testtype] if 'envpath' == testparam.lower(): for evparam in testparams['envpath']: #print ('tconf evparam = {}'.format(evparam)) for x,y in evparam.items(): if theplatform.lower() in util_functions.WinPlatforms: my_env[str(x)] = str('{}'.format(os.path.join('%s' % (my_env["PROGRAMFILES"]), y))) installpath = '{}/bin;{}'.format(y, installpath) print ('path for {}={}'.format(x, installpath), flush=True) else: my_env[str(x)] = str('{}'.format(os.path.join('%s' % (currdir), y))) installpath = '{}/bin;{}'.format(y, installpath) print ('path for {}={}'.format(x, installpath), flush=True) if 'envrelpath' == testparam.lower(): for x,y in testparams['envrelpath'].items(): my_env[str(x)] = str('{}'.format(os.path.join('%s' % (currdir), y))) installpath = '{}/bin;{}'.format(y, installpath) print ('path for {}={}'.format(x, installpath), flush=True) if 'env' == testparam.lower(): envitems = testparams['env'] for envitem in envitems: k = envitem v = envitems[k] my_env[str(k)] = str(v) ############################################## # check compiler env ############################################## if toolset != 'default' and 'XX' != configToolset: if not theplatform.lower() in util_functions.WinPlatforms: # now set any env vars for the compilers for compiler in compilers: k = compiler if not 'MPI' in compiler: if 'MPI' in buildconfig: k = 'MPI{}'.format(compiler) for compver in compilervers: v = compver['compiler'] m = compver['module'] if k == v and m == ts_mod: if 'env' in compver: x = compver['env'] y = compver['command'] my_env[str(x)] = str(y) if theplatform.lower() in util_functions.WinPlatforms: my_env["PATH"] = str('{}/bin;{}'.format(installpath, my_env["PATH"])) if len(targetnames) > 0: for target in targetnames: ###################################################### # Do configure command ##################################################### thereturncode = 1 print(os.listdir('.'), flush=True) starttime = time.time() startdatetime = datetime.datetime.now() buildext = '{}'.format(ctparams) buildsite = 'https://cdash.hdfgroup.org' buildoption = '' buildproject = '' buildgroup = 'Ant' for testoptions in yaml_data['configfile'][buildconfig][ctparams]['testoptions']: k = testoptions v = yaml_data['configfile'][buildconfig][ctparams]['testoptions'][k] if 'cdashdropsite' == k.lower(): buildsite = '{}'.format(v) if 'cdashgroup' == k.lower(): buildgroup = '{}'.format(v) if 'cdashproject' == k.lower(): buildproject = '{}'.format(v) if 'cdashoption' == k.lower(): buildoption = '{}'.format(v) if 'nameext' == k.lower(): buildext = '{}'.format(v) buildext = '{}[{}]_{}'.format(buildext, target, toolset) print(os.listdir('..'), flush=True) os.chdir(builddir) #### cdashconfname = 'HDFcdash.json' try: cdash_data = cdash_functions.cdash_config(buildconfig, slaveos, buildsite, buildproject, buildgroup, buildext) with open(cdashconfname, 'w') as json_file: json.dump(cdash_data, json_file) except Exception as e: print(e) shellstate = False if theplatform.lower() in util_functions.WinPlatforms: my_env["PATH"] = str('{};{}'.format(installpath, my_env["PATH"])) shellstate = True try: output = '' print ('command=ant -version:') test = subprocess.Popen(['ant', '-version'], shell=shellstate, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env, universal_newlines=True) (textout, texterr) = test.communicate() thereturncode = test.returncode if textout is not None: output = '{}{}\n'.format(output, textout) if texterr is not None: output = '{} \n{}'.format(output, texterr) print ('---- ant is {} '.format(output), flush=True) except Exception as e: thereturncode = 255 print(e) try: endtime = time.time() enddatetime = datetime.datetime.now() cdash_functions.upload_config_cdash(builddir, output, thereturncode, 'command=ant -version:', starttime, startdatetime, endtime, enddatetime) except Exception as e: print('Upload configure Exception: {}'.format(e), flush=True) thereturncode = 255 ###################################################### # Do build and test command ##################################################### print ('command=ant -buildfile {}/build.xml {}'.format(source_dir, target), flush=True) starttime = time.time() startdatetime = datetime.datetime.now() output = '' textout = '' texterr = '' try: test = subprocess.Popen(['ant', '-buildfile', '{}/build.xml'.format(source_dir), target], shell=shellstate, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env, universal_newlines=True) (textout, texterr) = test.communicate() thereturncode = test.returncode except Exception as e: print('subprocess build Exception: {}'.format(e), flush=True) thereturncode = 253 try: if os.path.exists('{}/antoutput.log'.format(builddir)): # with open('{}/antoutput.log'.format(builddir), 'r') as fp: # output = fp.read() os.rename('{}/antoutput.log'.format(builddir), '{}/{}output.log'.format(builddir, target)) if textout is not None: output = '{}{}\n'.format(output, textout) if texterr is not None and len(texterr) > 0: output = '{} \n{}'.format(output, texterr) except Exception as e: print('build file operation Exception: {}'.format(e), flush=True) thereturncode = 254 try: endtime = time.time() enddatetime = datetime.datetime.now() cdash_functions.upload_cdash(builddir, output, thereturncode, target, starttime, startdatetime, endtime, enddatetime) except Exception as e: print('Upload build Exception: {}'.format(e), flush=True) thereturncode = 255 if 'pack' in target.lower(): if publish: hdfbld = os.path.join(source_dir, 'build', 'dist') binary_dict = util_functions.format_binary_name_dict(theplatform, yaml_data['configfile'], buildconfig, ctparams, buildsys, compilername, theossize, True, False) for qatestpath, binary_fname in binary_dict.items(): if toolset != 'default': binary_fname = '{}-{}'.format(binary_fname, ts_path) print ('pull {} from {}'.format(binary_fname, hdfbld), flush=True) binary_file = binary_fname + util_functions.get_binary_ext(theplatform) print ('publish {} to {}'.format(binary_file, qatestpath), flush=True) currdir=os.getcwd() print(os.listdir('.'), flush=True) try: for file in os.listdir('{}'.format(hdfbld)): print ('file={}'.format(file), flush=True) if file.startswith('HDF'): if theplatform.lower() in util_functions.WinPlatforms: if file.endswith('.zip'): copy2 (os.path.join('%s' % (hdfbld), file), os.path.join('%s' % (builddir), binary_file)) os.chmod(os.path.join('%s' % (builddir), binary_file), 0o777) print ('copied: {}'.format(file), flush=True) else: if file.endswith('.gz'): copy2 (os.path.join('%s' % (hdfbld), file), os.path.join('%s' % (builddir), binary_file)) os.chmod(os.path.join('%s' % (builddir), binary_file), 0o777) print ('copied: {}'.format(file), flush=True) print ('file {} : done'.format(source_dir), flush=True) except Exception as e: print('copy2 compressed binary Exception: {}'.format(e), flush=True) thereturncode = 255 os.chdir(currdir) json_bbparams_data.append({ctparams+'-binary':binary_file}) print('copy build tree from {} to {}'.format(source_dir,builddir), flush=True) # copy hdfsrc/build to current hdfbld folder try: util_functions.copytree(os.path.join(source_dir,'build'),os.path.join(builddir,'build')) except Exception as e: print('copytree Exception: {}'.format(e), flush=True) thereturncode = 255 else: try: starttime = time.time() startdatetime = datetime.datetime.now() output = 'Build skipped' endtime = time.time() enddatetime = datetime.datetime.now() cdash_functions.upload_config_cdash(builddir, output, thereturncode, project_configureFlags, starttime, startdatetime, endtime, enddatetime) except Exception as e: print('Upload build Exception: {}'.format(e), flush=True) thereturncode = 255 json_bbparams_data.append(script_res) os.chdir(scriptdir) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) os.chdir(scriptdir) thereturncode = 2 script_tc_res = {} ############################################## # uninstall ftpbinary ############################################## for btparam in yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]: if 'envparams' in btparam: envparams = yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]['envparams'] #print ('tconf env == {}'.format(envparams)) if 'ftpbinary' in envparams: if envparams['ftpbinary']: script_tc_res.update({btparam+'-uninstallfolder':install_folder}) json_bbparams_data.append(script_tc_res) os.chdir(currinstalldir) #### currdir=os.getcwd() #### print ('current uninstall={}'.format(currdir), flush=True) thereturncode = step_functions.install_binary(json_bbparams_data, buildsys, theplatform, install_folder, 'uninstall') os.chdir(currdir) #### if thereturncode != 0: thescriptreturncode = 255 script_tc_res = {ctparams+'-'+testtype+'-'+script_name+'-'+toolset: 'FAILURE'} json_bbparams_data.append(script_tc_res) script_res = {ctparams+'-'+testtype+'-'+script_name+'-'+toolset: 'FAILURE'} json_bbparams_data.append(script_res) os.chdir(scriptdir) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) else: script_tc_res = {ctparams+'-'+testtype+'-'+script_name+'-'+toolset: 'SUCCESS'} json_bbparams_data.append(script_tc_res) if toolset != 'default' and not 'mingw' in toolse and not 'aarch' in toolset and 'XX' != configToolset: if theplatform.lower() not in util_functions.WinPlatforms: util_functions.module('purge') if thescriptreturncode != 0: script_tc_res = {ctparams+'-'+testtype+'-'+script_name: 'FAILURE'} json_bbparams_data.append(script_tc_res) script_res = {ctparams+'-'+script_name: 'FAILURE'} json_bbparams_data.append(script_res) os.chdir(scriptdir) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) else: script_tc_res = {ctparams+'-'+testtype+'-'+script_name: 'SUCCESS'} json_bbparams_data.append(script_tc_res) ############################################## # write final script status ############################################## script_res = {ctparams+'-'+script_name: 'SUCCESS'} json_bbparams_data.append(script_res) os.chdir(scriptdir) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) if thescriptreturncode != 0: raise Exception('ANT process failed - {}'.format(thescriptreturncode)) return thescriptreturncode if __name__ == '__main__': main(sys.argv)