#!/usr/bin/env python3 import sys import os import json import requests import time import datetime from io import BytesIO from io import StringIO import xml.sax.saxutils from . import util_functions from . import log_parse def cdash_config(configname, workeros, buildsite, buildproject, buildgroup, buildext): cdash_configurevars = {'url': '{}/submit.php'.format(buildsite)} cdash_configurevars.update({'project': '{}'.format(buildproject)}) cdash_configurevars.update({'buildname': '{}-{} {}'.format(buildproject, buildext, configname)}) cdash_configurevars.update({'sitename': '{}'.format(workeros)}) cdash_configurevars.update({'buildstamp': '{}-{}'.format(time.strftime("%Y%m%d-%H%M"), buildgroup)}) return cdash_configurevars def upload_config_cdash(toppath, outputdata, cmdret, configurelist, starttime, startdatetime, endtime, enddatetime): with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) url = json_cdash['url'] payload = {'project': json_cdash['project'], 'FileName': 'Configure.xml'} cdashfile = open('Configure.xml', 'w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('"') for item in configurelist: cdashfile.write('{}'.format(item)) cdashfile.write('"\n') cdashfile.write('\n') if len(outputdata) > 0: print('outputdata type={}'.format(type(outputdata))) try: outdata = xml.sax.saxutils.escape(outputdata) cdashfile.write(outdata) except Exception as e: print('xml.sax.saxutils Exception: = {}'.format(e)) cdashfile.write('\n') cdashfile.write('{}\n'.format(cmdret)) cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml"} print('upload configure') mydata = '' with open('Configure.xml', 'r') as fh: try: mydata = fh.read().encode('utf-8') except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('url={} resp.status_code={}'.format(url, resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/configure: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err)) def upload_make_cdash(toppath, outputdata, cmdret, buildlist, starttime, startdatetime, endtime, enddatetime): with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) url = json_cdash['url'] payload = {'project': json_cdash['project'], 'FileName': 'Build.xml'} cdashfile = open('Build.xml','w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('"') for item in buildlist: cdashfile.write('{}'.format(item)) cdashfile.write('"\n') print('parse_log_events') errors = [] warnings = [] if len(outputdata) > 0: print('outputdata type={}'.format(type(outputdata))) try: outdata = xml.sax.saxutils.escape(outputdata) errors, warnings = log_parse.parse_log_events(StringIO(outdata)) except Exception as e: print('parse_log_events Exception: = {}'.format(e)) nerr = len(errors) nwar = len(warnings) print('parse_log_events nerr={} nwar={}'.format(nerr,nwar)) if nerr > 0: for error in errors: try: cdashfile.write('\n') cdashfile.write('{}\n'.format(error.line_no)) cdashfile.write('{}\n'.format(error.text)) cdashfile.write('{}\n'.format('\n'.join(error.pre_context))) cdashfile.write('{}\n'.format('\n'.join(error.post_context))) cdashfile.write('0\n') except Exception as e: print('Error Exception: = {}'.format(e)) if nwar > 0: for warning in warnings: try: cdashfile.write('\n') cdashfile.write('{}\n'.format(warning.line_no)) cdashfile.write('{}\n'.format(warning.text)) cdashfile.write('{}\n'.format('\n'.join(warning.pre_context))) cdashfile.write('{}\n'.format('\n'.join(warning.post_context))) cdashfile.write('0\n') except Exception as e: print('Warning Exception: = {}'.format(e)) cdashfile.write('\n') cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml;charset=UTF-8"} print('upload build') mydata = '' with open('Build.xml', 'r') as fh: try: mydata = fh.read().encode('utf-8') except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('url={} resp.status_code={}'.format(url, resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/build: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err)) def upload_check_cdash(toppath, cmdret, testlist, starttime, startdatetime, endtime, enddatetime): dir_list = util_functions.get_filepaths(toppath) testnames = [] logfiles = [] parts = [] for f in dir_list: if f.endswith("chklog"): parts = os.path.split(f) logfiles.append(f) #print ('{} with {} - {}'.format(f,parts[0],parts[1])) testname, file_extension = os.path.splitext(parts[1]) testnames.append (testname) with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) url = json_cdash['url'] payload = {'project': json_cdash['project'], 'FileName': 'Test.xml'} cdashfile = open('Test.xml','w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('') for item in testnames: cdashfile.write('"') cdashfile.write('{} '.format(item)) cdashfile.write('"\n') cdashfile.write('\n') for item in logfiles: parts = os.path.split(item) testname, file_extension = os.path.splitext(parts[1]) #print ('test = {}'.format(testname)) chkexename = '{}.chkexe'.format(testname) #print ('test = {}'.format(chkexename)) chkexepath = os.path.join(parts[0], chkexename) #print ('test = {}'.format(chkexepath)) testresult = 'failed' if os.path.exists(chkexepath): testresult = 'passed' cdashfile.write('\n'.format(testresult)) cdashfile.write('{}\n'.format(testname)) cdashfile.write('{}\n'.format(parts[0])) testcommand = os.path.join(parts[0], testname) #print ('test = {}'.format(testcommand)) cdashfile.write('{}\n'.format(testcommand)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') teststarttime = util_functions.modification_date(testcommand) testendtime = util_functions.modification_date(item) cdashfile.write('{}\n'.format((testendtime - teststarttime).total_seconds())) cdashfile.write('\n') with open(item, 'r') as f: for line in f: try: cdashfile.write(xml.sax.saxutils.escape(line)) except Exception as e: print('Write Measurement Value Exception: = {}'.format(e)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('>\n') cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml"} print('upload test') mydata = '' with open('Test.xml', 'r') as fh: try: mydata = fh.read().encode('utf-8') except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('resp.status_code={}'.format(resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/Test: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err)) def upload_cdash(toppath, outputdata, cmdret, target, starttime, startdatetime, endtime, enddatetime): with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) mydata = '' url = json_cdash['url'] if target in ['package','packageSource']: payload = {'project': json_cdash['project'], 'FileName': 'Build.xml'} cdashfile = open('Build.xml','w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('"') cdashfile.write('{} '.format(target)) cdashfile.write('"\n') if cmdret > 0: cdashfile.write('\n') else: cdashfile.write('\n') cdashfile.write(xml.sax.saxutils.escape(outputdata)) cdashfile.write('\n') cdashfile.write('0\n') if cmdret > 0: cdashfile.write('\n') else: cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml"} print('upload build') with open('Build.xml', 'r') as fh: try: mydata = fh.read() except Exception as e: print('Read File Exception: = {}'.format(e)) else: payload = {'project': json_cdash['project'], 'FileName': 'Test.xml'} cdashfile = open('Test.xml','w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('') cdashfile.write('"') cdashfile.write('{} '.format(target)) cdashfile.write('"\n') cdashfile.write('\n') testresult = 'failed' if cmdret == 0: testresult = 'passed' cdashfile.write('\n'.format(testresult)) cdashfile.write('{}\n'.format(target)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write(xml.sax.saxutils.escape(outputdata)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('>\n') cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml"} print('upload test') with open('Test.xml', 'r') as fh: try: mydata = fh.read().encode('utf-8') except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('url={} resp.status_code={}'.format(url, resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/build: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err)) def upload_info_cdash(toppath, outputdata, cmdret, configurelist, starttime, startdatetime, endtime, enddatetime): with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) url = json_cdash['url'] payload = {'project': json_cdash['project'], 'FileName': 'Configure.xml'} cdashfile = open('Configure.xml', 'w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('"') for item in configurelist: cdashfile.write('{}'.format(item)) cdashfile.write('"\n') cdashfile.write('\n') if len(outputdata) > 0: print('outputdata type={}'.format(type(outputdata))) try: #cdashfile.write(xml.sax.saxutils.escape(BytesIO(outputdata.encode()))) cdashfile.write(xml.sax.saxutils.escape(outputdata)) except Exception as e: print('xml.sax.saxutils Exception: = {}'.format(e)) cdashfile.write('\n') cdashfile.write('{}\n'.format(cmdret)) cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml"} print('upload spack info') mydata = '' with open('Configure.xml', 'r') as fh: try: mydata = fh.read().encode('utf-8') except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('url={} resp.status_code={}'.format(url, resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/configure: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err)) def upload_spack_cdash(toppath, outputdata, cmdret, buildlist, starttime, startdatetime, endtime, enddatetime): with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) url = json_cdash['url'] payload = {'project': json_cdash['project'], 'FileName': 'Build.xml'} cdashfile = open('Build.xml','w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('"') for item in buildlist: cdashfile.write('{}'.format(item)) cdashfile.write('"\n') print('parse_log_events') errors = [] warnings = [] if len(outputdata) > 0: print('outputdata type={}'.format(type(outputdata))) try: errors, warnings = log_parse.parse_log_events(StringIO(outputdata)) except Exception as e: print('parse_log_events Exception: = {}'.format(e)) nerr = len(errors) nwar = len(warnings) print('nerr={} nwar={}'.format(nerr,nwar)) if nerr > 0: for error in errors: try: cdashfile.write('\n') cdashfile.write('{}\n'.format(error.line_no)) cdashfile.write('{}\n'.format(error.text)) cdashfile.write('{}\n'.format(error.pre_context)) cdashfile.write('{}\n'.format(error.post_context)) cdashfile.write('0\n') except Exception as e: print('Error Exception: = {}'.format(e)) if nwar > 0: for warning in warnings: try: cdashfile.write('\n') cdashfile.write('{}\n'.format(warning.line_no)) cdashfile.write('{}\n'.format(warning.text)) cdashfile.write('{}\n'.format(warning.pre_context)) cdashfile.write('{}\n'.format(warning.post_context)) cdashfile.write('0\n') except Exception as e: print('Warning Exception: = {}'.format(e)) cdashfile.write('\n') cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml;charset=UTF-8"} print('upload spack build') mydata = '' with open('Build.xml', 'r') as fh: try: mydata = fh.read() except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('url={} resp.status_code={}'.format(url, resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/build: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err)) def upload_xml_cdash(toppath, xmldata, cmdret, target, starttime, startdatetime, endtime, enddatetime): with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file: json_cdash = json.load(json_file) mydata = '' url = json_cdash['url'] payload = {'project': json_cdash['project'], 'FileName': 'Test.xml'} cdashfile = open('Test.xml','w') cdashfile.write('\n') cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp'])) cdashfile.write('\n') cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(starttime)) cdashfile.write('') cdashfile.write('"') cdashfile.write('{} '.format(target)) cdashfile.write('"\n') cdashfile.write('\n') testresult = 'failed' if cmdret == 0: testresult = 'passed' cdashfile.write('\n'.format(testresult)) cdashfile.write('{}\n'.format(target)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write(xml.sax.saxutils.escape(xmldata)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('>\n') cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z"))) cdashfile.write('{}\n'.format(endtime)) cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0)) cdashfile.write('\n') cdashfile.write('\n') cdashfile.write('\n') cdashfile.close() # Create your header as required #headers = {"content-type": "application/xml"} print('upload test') with open('Test.xml', 'r') as fh: try: mydata = fh.read().encode('utf-8') except Exception as e: print('Read File Exception: = {}'.format(e)) try: resp = requests.put(url, data=mydata, params=payload) #, headers=headers) print('url={} resp.status_code={}'.format(url, resp.status_code)) if resp.status_code != 200: print ('FAILED PUT /builders/build: {}'.format(resp.status_code)) # This means something went wrong. if resp.status_code != 504: # Gateway timeout raise Exception('PUT /builders/ {}'.format(resp.status_code)) except Exception as err: print('Requests Exception: = {}'.format(err))