#!/usr/bin/env python3 import sys import os import json import hashlib import paramiko import datetime import zipfile, tarfile from shutil import copy2 def getPlatforms(buildsys): scriptdir=os.getcwd() #### jsbbsystems_file = 'bbsystems.json' if buildsys != 'build': if scriptdir.split(os.sep)[-1] == buildsys: jsbbsystems_file = '../bbsystems.json' with open(jsbbsystems_file, 'r') as json_bbsystems_file: json_bbsystems_data = json.load(json_bbsystems_file) global Win10IntelPlatforms Win10IntelPlatforms = list(json_bbsystems_data['systems']['Win10IntelPlatforms']) global Win10Platforms Win10Platforms = list(json_bbsystems_data['systems']['Win10Platforms']) global Win11Platforms Win11Platforms = list(json_bbsystems_data['systems']['Win11Platforms']) global Win10MpiPlatforms Win10MpiPlatforms = list(json_bbsystems_data['systems']['Win10MpiPlatforms']) global WinPlatforms WinPlatforms = list(Win10Platforms) WinPlatforms.extend(list(Win11Platforms)) WinPlatforms.extend(list(Win10IntelPlatforms)) WinPlatforms.extend(list(Win10MpiPlatforms)) global LinuxPlatforms LinuxPlatforms = json_bbsystems_data['systems']['LinuxPlatforms'] global MacPlatforms MacPlatforms = json_bbsystems_data['systems']['MacPlatforms'] global BuilderTypes BuilderTypes = json_bbsystems_data['systems']['buildertypes'] global CodeBuildTypes CodeBuildTypes = json_bbsystems_data['systems']['CodeBuildTypes'] def VS_full_to_short(argument, ossize): print(argument) switcher = { "Visual Studio 14 2015": "32-vs14", "Visual Studio 14 2015 Win64": "64-vs14", "Visual Studio 15 2017": "32-vs15", "Visual Studio 15 2017 Win64": "64-vs15", } if '2019' in argument: if ossize == '32': return "32-vs16" else: return "64-vs16" elif '2022' in argument: if ossize == '32': return "32-vs17" else: return "64-vs17" else: return switcher.get(argument, "Ninja") def VS_full_to_VScompile(argument, ossize): #print(argument) switcher = { "Visual Studio 14 2015": "32_VS2015", "Visual Studio 14 2015 Win64": "64_VS2015", "Visual Studio 15 2017": "32_VS2017", "Visual Studio 15 2017 Win64": "64_VS2017", "Visual Studio 16 2019": "{}_VS2019".format(ossize), "Visual Studio 17 2022": "{}_VS2022".format(ossize), } return switcher.get(argument, "Ninja") def linux_to_binary(basename, platform, ossize): switcher = { "fedora35": "fedora35", "fedora36": "fedora36", "fedora37": "fedora37", "fedora38": "fedora38", "fedora39": "fedora39", "fedora40": "fedora40", "fedora41": "fedora41", "fedora42": "fedora42", "fedora43": "fedora43", "debian86": "debian86", "ubuntu2004": "ubuntu2004", "ubuntu2010": "ubuntu2010", "ubuntu2104": "ubuntu2104", "ubuntu2110": "ubuntu2110", "ubuntu2204": "ubuntu2204", "ubuntu2210": "ubuntu2210", "ubuntu2404": "ubuntu2404", "ubuntu2410": "ubuntu2410", "ubuntu2504": "ubuntu2504", "ubuntu2510": "ubuntu2510", "mingw2204": "mingw2204", "mingw2404": "mingw2404", "aarch2204": "aarch2204", "aarch2404": "aarch2404", "suse152": "suse152", "suse153": "suse153", "suse154": "suse154", "centos6": "centos6", "centos7": "centos7", "centos8": "centos8", "rocky9": "rocky9", "rocky10": "rocky10", "cygwin": "cygwin", "ppc8beel7": "ppc8beel7", "ppc8leel7": "ppc8leel7", "ppc9el7": "ppc9el7", "osx1013": "osx1013", "osx1014": "osx1014", "osx1015": "osx1015", "macos11m1": "macos11m1", "macos12m1": "macos12m1", "macos13m1": "macos13m1", "macos14m1": "macos14m1", "macos11": "macos11", "macos12": "macos12", "macos13": "macos13", "macos14": "macos14", "macos15": "macos15", } binname = basename % switcher.get(platform, "linux") return binname+'_'+ossize def create_sftp_client(host, port, username, password, keyfilepath, keyfiletype, usessh): """ create_sftp_client(host, port, username, password, keyfilepath, keyfiletype) -> SFTPClient Creates a SFTP client connected to the supplied host on the supplied port authenticating as the user with supplied username and supplied password or with the private key in a file with the supplied path. If a private key is used for authentication, the type of the keyfile needs to be specified as DSA or RSA. :rtype: SFTPClient object. """ ssh = None sftp = None key = None transport = None try: if keyfilepath is not None: # Get private key used to authenticate user. if keyfiletype == 'DSA': # The private key is a DSA type key. key = paramiko.DSSKey.from_private_key_file(keyfilepath) else: # The private key is a RSA type key. key = paramiko.RSAKey.from_private_key_file(keyfilepath) print ('retrieved key') if usessh: # Connect SSH client accepting all host keys. ssh = paramiko.SSHClient() print ('SSHClient created') ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(host, port, username, password, key, disabled_algorithms={'pubkeys': ['rsa-sha2-256', 'rsa-sha2-512']}) print ('connected') # Using the SSH client, create a SFTP client. sftp = ssh.open_sftp() # Keep a reference to the SSH client in the SFTP client as to prevent the former from # being garbage collected and the connection from being closed. sftp.sshclient = ssh else: # Create Transport object using supplied method of authentication. transport = paramiko.Transport((host, port)) print ('transport created') transport.connect(None, username, password, key, disabled_algorithms={'pubkeys': ['rsa-sha2-256', 'rsa-sha2-512']}) print ('connected') sftp = paramiko.SFTPClient.from_transport(transport) if sftp is not None: print ('sftp okay') return sftp except Exception as e: print('An error occurred creating SFTP client: {}: {}'.format(e.__class__, e)) if sftp is not None: sftp.close() if transport is not None: transport.close() if ssh is not None: ssh.close() pass def module(command, *arguments): commands = os.popen('/usr/share/lmod/lmod/libexec/lmod python %s %s'\ % (command, ' '.join(arguments))).read() exec(commands) def copytree(src, dst): if not os.path.exists(dst): os.makedirs(dst) for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) if os.path.isdir(s): copytree(s, d) else: if not os.path.exists(d) or os.stat(s).st_mtime - os.stat(d).st_mtime > 1: copy2(s, d) def get_directory_structure(rootdir): """ Creates a nested dictionary that represents the folder structure of rootdir """ thedir = {} rootdir = rootdir.rstrip(os.sep) start = rootdir.rfind(os.sep) + 1 for path, dirs, files in os.walk(rootdir): folders = path[start:].split(os.sep) subdir = dict.fromkeys(files) parent = reduce(dict.get, folders[:-1], thedir) parent[folders[-1]] = subdir return thedir def cmake_to_binary(basename, platform, ossize): binname = basename % platform return binname+'_'+ossize def windows_to_binary(basename, platform, compilername): if platform.lower() in Win10MpiPlatforms: if platform.lower() in ['mpvsi10vs16']: binname = basename % 'win10-Intel_MPI' else: binname = basename % 'win10-MPI' elif platform.lower() in Win10Platforms: binname = basename % 'win10' elif platform.lower() in Win11Platforms: binname = basename % 'win11' else: binname = basename % platform return binname+'_'+compilername def modification_date(filename): t = os.path.getmtime(filename) return datetime.datetime.fromtimestamp(t) def get_filepaths(directory): """ This function will generate the file names in a directory tree by walking the tree either top-down or bottom-up. For each directory in the tree rooted at directory top (including top itself), it yields a 3-tuple (dirpath, dirnames, filenames). """ file_paths = [] # List which will store all of the full filepaths. # Walk the tree. for root, directories, files in os.walk(directory): for filename in files: # Join the two strings in order to form the full filepath. filepath = os.path.join(os.path.normpath(root), filename) file_paths.append(filepath) # Add it to the list. return file_paths # Self-explanatory. def checksum_md5(filename): md5 = hashlib.md5() with open(filename,'rb') as f: for chunk in iter(lambda: f.read(8192), b''): md5.update(chunk) return md5.hexdigest() def checksum_sha256(filename): sha_hash = hashlib.sha256() with open(filename,'rb') as f: for chunk in iter(lambda: f.read(8192), b''): sha_hash.update(chunk) return sha_hash.hexdigest() def format_binary_name_list(slaveplatform, config_data, buildconfig, ctparams, buildsys, compilername, ossize, binaryLib, binarySource): binary_list = [] binary_fname = '' qatestdir = '' print ('slaveplatform = {}'.format(slaveplatform)) for btparam in config_data[buildconfig][ctparams]['buildsys'][buildsys]: if 'envparams' in btparam: envparams = config_data[buildconfig][ctparams]['buildsys'][buildsys]['envparams'] print ('binaryLib == {} binarySource == {}'.format(binaryLib, binarySource)) for envparam in envparams: if bool(binaryLib): if 'ftpbinary' in envparam: for k,v in envparams['ftpbinary'].items(): if 'binaryfolder' not in k: binary_dict = {} qatestdir = k if slaveplatform.lower() in WinPlatforms: binary_fname = windows_to_binary(v, slaveplatform.lower(), compilername) qatestpath = qatestdir + 'windows' else: qatestpath = qatestdir + 'unix' if 'btctest' == buildsys or 'ctest' == buildsys: binary_fname = cmake_to_binary(v, slaveplatform.lower(), ossize) else: binary_fname = linux_to_binary(v, slaveplatform.lower(), ossize) print ('get binary is {}'.format(binary_fname)) binary_dict[qatestpath] = binary_fname binary_list.append(binary_dict) if 'ftpaltbinary' in envparam: for k,v in envparams['ftpaltbinary'].items(): if 'binaryfolder' not in k: binary_dict = {} qatestdir = k if slaveplatform.lower() in WinPlatforms: binary_fname = windows_to_binary(v, slaveplatform.lower(), compilername) qatestpath = qatestdir + 'windows' else: qatestpath = qatestdir + 'unix' if 'btctest' == buildsys or 'ctest' == buildsys: binary_fname = cmake_to_binary(v, slaveplatform.lower(), ossize) else: binary_fname = linux_to_binary(v, slaveplatform.lower(), ossize) print ('get binary is {}'.format(binary_fname)) binary_dict[qatestpath] = binary_fname binary_list.append(binary_dict) if bool(binarySource) and 'srcbinary' in envparam: for k,v in envparam['srcbinary'].items(): binary_dict = {} qatestdir = k if slaveplatform.lower() in WinPlatforms: binary_fname = windows_to_binary(v, slaveplatform.lower(), compilername) qatestpath = qatestdir + 'windows' else: qatestpath = qatestdir + 'unix' if 'btctest' == buildsys or 'ctest' == buildsys: binary_fname = cmake_to_binary(v, slaveplatform.lower(), ossize) else: binary_fname = linux_to_binary(v, slaveplatform.lower(), ossize) print ('getSource binary is {}'.format(binary_fname)) binary_dict[qatestpath] = binary_fname binary_list.append(binary_dict) return binary_list def format_binary_name_dict(slaveplatform, config_data, buildconfig, ctparams, buildsys, compilername, ossize, publish, publishSource): binary_dict = {} binary_fname = '' qatestdir = '' print ('slaveplatform = {}'.format(slaveplatform)) for btparam in config_data[buildconfig][ctparams]['buildsys'][buildsys]: if 'envparams' in btparam: envparams = config_data[buildconfig][ctparams]['buildsys'][buildsys]['envparams'] #print ('tconf env == {}'.format(envparams)) for envparam in envparams: if bool(publish): if 'ant' == buildsys: if 'antbinary' in envparam: for binfile in envparams['antbinary']: k = binfile v = envparams['antbinary'][k] qatestdir = k if slaveplatform.lower() in WinPlatforms: binary_fname = windows_to_binary(v, slaveplatform.lower(), compilername) qatestpath = qatestdir + 'windows' else: binary_fname = cmake_to_binary(v, slaveplatform.lower(), ossize) qatestpath = qatestdir + 'unix' print ('publish ant binary is {}'.format(binary_fname)) binary_dict[qatestpath] = binary_fname else: if 'pubbinary' in envparam: for binfile in envparams['pubbinary']: k = binfile v = envparams['pubbinary'][k] qatestdir = k if slaveplatform.lower() in WinPlatforms: binary_fname = windows_to_binary(v, slaveplatform.lower(), compilername) qatestpath = qatestdir + 'windows' else: if 'btctest' == buildsys or 'ctest' == buildsys: binary_fname = cmake_to_binary(v, slaveplatform.lower(), ossize) else: binary_fname = linux_to_binary(v, slaveplatform.lower(), ossize) qatestpath = qatestdir + 'unix' print ('publish binary is {}'.format(binary_fname)) binary_dict[qatestpath] = binary_fname if bool(publishSource): if 'srcbinary' in envparam: for binfile in envparams['srcbinary']: k = binfile v = envparams['srcbinary'][k] qatestdir = k if slaveplatform.lower() in WinPlatforms: binary_fname = windows_to_binary(v, slaveplatform.lower(), compilername) qatestpath = qatestdir + 'windows' else: if 'btctest' == buildsys or 'ctest' == buildsys: binary_fname = cmake_to_binary(v, slaveplatform.lower(), ossize) else: binary_fname = linux_to_binary(v, slaveplatform.lower(), ossize) qatestpath = qatestdir + 'unix' print ('publishSource binary is {}'.format(binary_fname)) binary_dict[qatestpath] = binary_fname return binary_dict def get_binary_ext(slaveplatform): if slaveplatform.lower() in WinPlatforms: binary_ext = '.zip' else: binary_ext = '.tar.gz' return binary_ext def extract_binary(binary_file): fileExtension = os.path.splitext(binary_file)[1][1:].strip().lower() if 'zip' == fileExtension: cf = zipfile.ZipFile(binary_file) cf.extractall() cf.close() print ('zip Done.') elif 'gz' == fileExtension: cf = tarfile.open(binary_file, 'r:gz') cf.extractall() cf.close() print ('gz Done.') else: cf = tarfile.open(binary_file, 'r') cf.extractall() cf.close() print ('other Done.')