#!/usr/bin/env python3 # # ftp get binary and uncompress import sys, json, yaml import os from os.path import expanduser from util_functions import util_functions from util_functions import step_functions #doyftpwinsign arguments # 1. HOST # 2. DIRN # 3. os of slave # 4. os size of slave # 5. platform of slave # 6. configuration # 7. ctparams # 8. build system # 9. local folder for file # 10. btconfig # 11. datstore def main(argv): print ('{}:{}:{}:{}:{}:{}:{}:{}:{}:{}:{}'.format( sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6],sys.argv[7], sys.argv[8],sys.argv[9],sys.argv[10],sys.argv[11])) HOST = sys.argv[1] DIRN = sys.argv[2] slaveos = sys.argv[3] theossize = sys.argv[4] theplatform = sys.argv[5] buildconfig = sys.argv[6] ctparams = sys.argv[7] buildsys = sys.argv[8] install_folder = sys.argv[9] configfile = sys.argv[10] datastore = {} datastore = eval(argv[11]) generator = datastore['generator'] generatorarch = datastore['gen_arch'] schedule = datastore['scheduler'] compilername = util_functions.VS_full_to_short(generator, theossize) script_path, script_name = os.path.split(sys.argv[0]) userhome = expanduser("~") bbuserhome = os.path.join(userhome, '.buildbot') util_functions.getPlatforms(buildsys) if theplatform.lower() in util_functions.WinPlatforms: if not 'override' in schedule.lower(): toolsets = {'default': ['default']} script_res = {ctparams+'-'+script_name: 'FAILURE'} scriptdir=os.getcwd() #### print ('scriptdir={}'.format(scriptdir)) jsparams_file = 'bb_params.json' if scriptdir.split(os.sep)[-1] == buildsys: jsparams_file = '../bb_params.json' with open(jsparams_file, 'r') as json_bbparams_file: json_bbparams_data = json.load(json_bbparams_file) with open(configfile, 'r') as yaml_file: yaml_data = yaml.safe_load(yaml_file) #print(yaml_data) currdir=os.getcwd() #### for theplatform in util_functions.WinPlatforms: thereturncode = 2 if buildsys in yaml_data['configfile'][buildconfig][ctparams]['buildsys']: target_binaries = [] target_binaries = util_functions.format_binary_name_list(theplatform, yaml_data['configfile'], buildconfig, ctparams, buildsys, compilername, theossize, True, False) #print ('target_binaries {}'.format(target_binaries)) configToolset = 'default' if 'MPI' in buildconfig: configToolset = 'MPI' for toolset in toolsets[configToolset]: print('toolset={}'.format(toolset)) ts_path = 'hdfbld' if toolset != 'default': ts_path = (toolset.split(',', 1))[0] ts_path = (ts_path.split(' ', 1))[0] else: if 'mingw' or 'aarch' in toolset: ts_path = (toolset.split(None, 1))[0] else: try: ts_path = (toolset.split('/', 1))[1] except: ts_path = (toolset.split(None, 1))[0] if not os.path.exists('{}'.format(ts_path)): os.mkdir(ts_path) thereturncode = step_functions.ftp_uncompress(json_bbparams_data, HOST, DIRN, theplatform, toolset, ts_path, install_folder, target_binaries) if thereturncode == 0: script_tc_res = {} for btparam in yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]: if 'envparams' in btparam: envparams = yaml_data['configfile'][buildconfig][ctparams]['buildsys'][buildsys]['envparams'] #print ('tconf env == {}'.format(envparams)) if thereturncode == 255: json_bbparams_data.append(script_res) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) raise Exception('Unable to get and uncompress file') ###################################################### # Do signing command ##################################################### thereturncode = 1 os.chdir(install_folder) print(os.listdir('.')) for file in os.listdir('{}'.format(install_folder)): if file.endswith('.msi') or file.endswith('.exe'): try: output = '' print ('command=signtool sign file={}'.format(file)) test = subprocess.Popen(['signtool', 'sign', '/tr', 'http://timestamp.digicert.com', '/fd', 'sha256', '/f', 'TheHDFGroup.cer', '/csp', "eToken Base Cryptographic Provider", '/kc', "[{{3poLARcub$}}]=Sectigo_20230615125101", '{}'.format(file)], shell=shellstate, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env, universal_newlines=True) (textout, texterr) = test.communicate() thereturncode = test.returncode if textout is not None: output = '{}{}\n'.format(output, textout) if texterr is not None: output = '{} \n{}'.format(output, texterr) print ('signing result is {} '.format(output)) except Exception as e: thereturncode = 255 print(e) binary_dict = util_functions.format_binary_name_dict(theplatform, yaml_data['configfile'], buildconfig, ctparams, buildsys, compilername, theossize, True, False) binary_ext = util_functions.get_binary_ext(theplatform) for qatestpath, binary_fname in binary_dict.items(): if toolset != 'default': binary_fname = '{}-{}'.format(binary_fname, ts_path) print ('push {} to {}'.format(binary_fname, qatestpath)) print ('binary_name={}'.format(binary_fname)) json_bbparams_data.append({ctparams+'-binary':binary_fname}) currdir=os.getcwd() if '.zip' == binary_ext: make_archive(binary_fname, 'zip', currdir, '{}'.format(install_folder)) print ('Done zip.') elif '.tar.gz' == binary_ext: make_archive(binary_fname, 'gztar', currdir, '{}'.format(install_folder)) print ('Done gz.') else: make_archive(binary_fname, 'tar', currdir, '{}'.format(install_folder)) print ('Done.') binary_file = binary_fname + util_functions.get_binary_ext(theplatform) os.chmod('{}'.format(binary_file), 0o777) os.chdir(currdir) print ('publish {} to {}'.format(binary_file, qatestpath)) if os.path.exists(binary_file): print ('upload {}'.format(binary_file)) filesize_b = os.path.getsize(binary_file) if filesize_b > 4096: binary_sha256 = '{}.sha256'.format(binary_file) binary_sha256_file = open(binary_sha256,'w') binary_sha256_file.write(util_functions.checksum_sha256(binary_file)) binary_sha256_file.close() sftpclient = util_functions.create_sftp_client(HOST, 22, 'hdftest', None, os.path.expanduser(os.path.join("~", ".ssh", "bitbidkey")), 'RSA', True) if sftpclient is not None: try: sftpclient.chdir( '{}{}'.format(DIRN,qatestpath)) print ('*** Changed to folder: "{}{}"'.format(DIRN,qatestpath)) except Exception as e: thereturncode = 255 print ('ERROR: cannot CD to "{}{}"'.format(DIRN,qatestpath)) if sftpclient is not None: sftpclient.close() currdir=os.getcwd() #### print ('current2={}'.format(currdir)) try: filename = '{}'.format(binary_file) print ('Putting {}'.format(filename)) sftpclient.put(filename, filename) print ('File {} uploaded'.format(filename)) except Exception as e: thereturncode = 255 print ('ERROR: cannot write file {}'.format(filename)) sftpclient.chmod(filename, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH) try: filename = '{}'.format(binary_sha256) print ('Putting {}'.format(filename)) sftpclient.put(filename, filename) print ('File {} uploaded'.format(filename)) except Exception as e: thereturncode = 255 print ('ERROR: cannot write file {}'.format(filename)) sftpclient.chmod(filename, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH) os.chdir(currdir) #### sftpclient.close() else: print ('Binary file {} size is too small'.format(binary_file)) thereturncode = 1 if thereturncode != 0: script_ts_res = {ctparams+'-'+script_name+'-'+toolset: 'FAILURE'} else: script_ts_res = {ctparams+'-'+script_name+'-'+toolset: 'SUCCESS'} json_bbparams_data.append(script_ts_res) os.chdir(currdir) script_res = {ctparams+'-'+script_name: 'SUCCESS'} else: script_res = {ctparams+'-'+script_name: 'SKIPPED'} json_bbparams_data.append(script_res) with open(jsparams_file, 'w') as json_bbparams_file: json.dump(json_bbparams_data, json_bbparams_file) return thereturncode if __name__ == '__main__': main(sys.argv)