#!/usr/bin/env python3
import sys
import os
import json
import requests
import time
import datetime
from io import BytesIO
from io import StringIO
import xml.sax.saxutils
from . import util_functions
from . import log_parse
def cdash_config(configname, workeros, buildsite, buildproject, buildgroup, buildext):
cdash_configurevars = {'url': '{}/submit.php'.format(buildsite)}
cdash_configurevars.update({'project': '{}'.format(buildproject)})
cdash_configurevars.update({'buildname': '{}-{} {}'.format(buildproject, buildext, configname)})
cdash_configurevars.update({'sitename': '{}'.format(workeros)})
cdash_configurevars.update({'buildstamp': '{}-{}'.format(time.strftime("%Y%m%d-%H%M"), buildgroup)})
return cdash_configurevars
def upload_config_cdash(toppath, outputdata, cmdret, configurelist, starttime, startdatetime, endtime, enddatetime):
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
url = json_cdash['url']
payload = {'project': json_cdash['project'], 'FileName': 'Configure.xml'}
cdashfile = open('Configure.xml', 'w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('"')
for item in configurelist:
cdashfile.write('{}'.format(item))
cdashfile.write('"\n')
cdashfile.write('\n')
if len(outputdata) > 0:
print('outputdata type={}'.format(type(outputdata)))
try:
outdata = xml.sax.saxutils.escape(outputdata)
cdashfile.write(outdata)
except Exception as e:
print('xml.sax.saxutils Exception: = {}'.format(e))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(cmdret))
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml"}
print('upload configure')
mydata = ''
with open('Configure.xml', 'r') as fh:
try:
mydata = fh.read().encode('utf-8')
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('url={} resp.status_code={}'.format(url, resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/configure: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))
def upload_make_cdash(toppath, outputdata, cmdret, buildlist, starttime, startdatetime, endtime, enddatetime):
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
url = json_cdash['url']
payload = {'project': json_cdash['project'], 'FileName': 'Build.xml'}
cdashfile = open('Build.xml','w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('"')
for item in buildlist:
cdashfile.write('{}'.format(item))
cdashfile.write('"\n')
print('parse_log_events')
errors = []
warnings = []
if len(outputdata) > 0:
print('outputdata type={}'.format(type(outputdata)))
try:
outdata = xml.sax.saxutils.escape(outputdata)
errors, warnings = log_parse.parse_log_events(StringIO(outdata))
except Exception as e:
print('parse_log_events Exception: = {}'.format(e))
nerr = len(errors)
nwar = len(warnings)
print('parse_log_events nerr={} nwar={}'.format(nerr,nwar))
if nerr > 0:
for error in errors:
try:
cdashfile.write('\n')
cdashfile.write('{}\n'.format(error.line_no))
cdashfile.write('{}\n'.format(error.text))
cdashfile.write('{}\n'.format('\n'.join(error.pre_context)))
cdashfile.write('{}\n'.format('\n'.join(error.post_context)))
cdashfile.write('0\n')
except Exception as e:
print('Error Exception: = {}'.format(e))
if nwar > 0:
for warning in warnings:
try:
cdashfile.write('\n')
cdashfile.write('{}\n'.format(warning.line_no))
cdashfile.write('{}\n'.format(warning.text))
cdashfile.write('{}\n'.format('\n'.join(warning.pre_context)))
cdashfile.write('{}\n'.format('\n'.join(warning.post_context)))
cdashfile.write('0\n')
except Exception as e:
print('Warning Exception: = {}'.format(e))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml;charset=UTF-8"}
print('upload build')
mydata = ''
with open('Build.xml', 'r') as fh:
try:
mydata = fh.read().encode('utf-8')
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('url={} resp.status_code={}'.format(url, resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/build: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))
def upload_check_cdash(toppath, cmdret, testlist, starttime, startdatetime, endtime, enddatetime):
dir_list = util_functions.get_filepaths(toppath)
testnames = []
logfiles = []
parts = []
for f in dir_list:
if f.endswith("chklog"):
parts = os.path.split(f)
logfiles.append(f)
#print ('{} with {} - {}'.format(f,parts[0],parts[1]))
testname, file_extension = os.path.splitext(parts[1])
testnames.append (testname)
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
url = json_cdash['url']
payload = {'project': json_cdash['project'], 'FileName': 'Test.xml'}
cdashfile = open('Test.xml','w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('')
for item in testnames:
cdashfile.write('"')
cdashfile.write('{} '.format(item))
cdashfile.write('"\n')
cdashfile.write('\n')
for item in logfiles:
parts = os.path.split(item)
testname, file_extension = os.path.splitext(parts[1])
#print ('test = {}'.format(testname))
chkexename = '{}.chkexe'.format(testname)
#print ('test = {}'.format(chkexename))
chkexepath = os.path.join(parts[0], chkexename)
#print ('test = {}'.format(chkexepath))
testresult = 'failed'
if os.path.exists(chkexepath):
testresult = 'passed'
cdashfile.write('\n'.format(testresult))
cdashfile.write('{}\n'.format(testname))
cdashfile.write('{}\n'.format(parts[0]))
testcommand = os.path.join(parts[0], testname)
#print ('test = {}'.format(testcommand))
cdashfile.write('{}\n'.format(testcommand))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
teststarttime = util_functions.modification_date(testcommand)
testendtime = util_functions.modification_date(item)
cdashfile.write('{}\n'.format((testendtime - teststarttime).total_seconds()))
cdashfile.write('\n')
with open(item, 'r') as f:
for line in f:
try:
cdashfile.write(xml.sax.saxutils.escape(line))
except Exception as e:
print('Write Measurement Value Exception: = {}'.format(e))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('>\n')
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml"}
print('upload test')
mydata = ''
with open('Test.xml', 'r') as fh:
try:
mydata = fh.read().encode('utf-8')
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('resp.status_code={}'.format(resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/Test: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))
def upload_cdash(toppath, outputdata, cmdret, target, starttime, startdatetime, endtime, enddatetime):
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
mydata = ''
url = json_cdash['url']
if target in ['package','packageSource']:
payload = {'project': json_cdash['project'], 'FileName': 'Build.xml'}
cdashfile = open('Build.xml','w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('"')
cdashfile.write('{} '.format(target))
cdashfile.write('"\n')
if cmdret > 0:
cdashfile.write('\n')
else:
cdashfile.write('\n')
cdashfile.write(xml.sax.saxutils.escape(outputdata))
cdashfile.write('\n')
cdashfile.write('0\n')
if cmdret > 0:
cdashfile.write('\n')
else:
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml"}
print('upload build')
with open('Build.xml', 'r') as fh:
try:
mydata = fh.read()
except Exception as e:
print('Read File Exception: = {}'.format(e))
else:
payload = {'project': json_cdash['project'], 'FileName': 'Test.xml'}
cdashfile = open('Test.xml','w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('')
cdashfile.write('"')
cdashfile.write('{} '.format(target))
cdashfile.write('"\n')
cdashfile.write('\n')
testresult = 'failed'
if cmdret == 0:
testresult = 'passed'
cdashfile.write('\n'.format(testresult))
cdashfile.write('{}\n'.format(target))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write(xml.sax.saxutils.escape(outputdata))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('>\n')
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml"}
print('upload test')
with open('Test.xml', 'r') as fh:
try:
mydata = fh.read().encode('utf-8')
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('url={} resp.status_code={}'.format(url, resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/build: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))
def upload_info_cdash(toppath, outputdata, cmdret, configurelist, starttime, startdatetime, endtime, enddatetime):
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
url = json_cdash['url']
payload = {'project': json_cdash['project'], 'FileName': 'Configure.xml'}
cdashfile = open('Configure.xml', 'w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('"')
for item in configurelist:
cdashfile.write('{}'.format(item))
cdashfile.write('"\n')
cdashfile.write('\n')
if len(outputdata) > 0:
print('outputdata type={}'.format(type(outputdata)))
try:
#cdashfile.write(xml.sax.saxutils.escape(BytesIO(outputdata.encode())))
cdashfile.write(xml.sax.saxutils.escape(outputdata))
except Exception as e:
print('xml.sax.saxutils Exception: = {}'.format(e))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(cmdret))
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml"}
print('upload spack info')
mydata = ''
with open('Configure.xml', 'r') as fh:
try:
mydata = fh.read().encode('utf-8')
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('url={} resp.status_code={}'.format(url, resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/configure: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))
def upload_spack_cdash(toppath, outputdata, cmdret, buildlist, starttime, startdatetime, endtime, enddatetime):
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
url = json_cdash['url']
payload = {'project': json_cdash['project'], 'FileName': 'Build.xml'}
cdashfile = open('Build.xml','w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('"')
for item in buildlist:
cdashfile.write('{}'.format(item))
cdashfile.write('"\n')
print('parse_log_events')
errors = []
warnings = []
if len(outputdata) > 0:
print('outputdata type={}'.format(type(outputdata)))
try:
errors, warnings = log_parse.parse_log_events(StringIO(outputdata))
except Exception as e:
print('parse_log_events Exception: = {}'.format(e))
nerr = len(errors)
nwar = len(warnings)
print('nerr={} nwar={}'.format(nerr,nwar))
if nerr > 0:
for error in errors:
try:
cdashfile.write('\n')
cdashfile.write('{}\n'.format(error.line_no))
cdashfile.write('{}\n'.format(error.text))
cdashfile.write('{}\n'.format(error.pre_context))
cdashfile.write('{}\n'.format(error.post_context))
cdashfile.write('0\n')
except Exception as e:
print('Error Exception: = {}'.format(e))
if nwar > 0:
for warning in warnings:
try:
cdashfile.write('\n')
cdashfile.write('{}\n'.format(warning.line_no))
cdashfile.write('{}\n'.format(warning.text))
cdashfile.write('{}\n'.format(warning.pre_context))
cdashfile.write('{}\n'.format(warning.post_context))
cdashfile.write('0\n')
except Exception as e:
print('Warning Exception: = {}'.format(e))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml;charset=UTF-8"}
print('upload spack build')
mydata = ''
with open('Build.xml', 'r') as fh:
try:
mydata = fh.read()
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('url={} resp.status_code={}'.format(url, resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/build: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))
def upload_xml_cdash(toppath, xmldata, cmdret, target, starttime, startdatetime, endtime, enddatetime):
with open('{}/HDFcdash.json'.format(toppath), 'r') as json_file:
json_cdash = json.load(json_file)
mydata = ''
url = json_cdash['url']
payload = {'project': json_cdash['project'], 'FileName': 'Test.xml'}
cdashfile = open('Test.xml','w')
cdashfile.write('\n')
cdashfile.write('\n'.format(json_cdash['buildname'], json_cdash['sitename'], json_cdash['buildstamp']))
cdashfile.write('\n')
cdashfile.write('{}\n'.format(startdatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(starttime))
cdashfile.write('')
cdashfile.write('"')
cdashfile.write('{} '.format(target))
cdashfile.write('"\n')
cdashfile.write('\n')
testresult = 'failed'
if cmdret == 0:
testresult = 'passed'
cdashfile.write('\n'.format(testresult))
cdashfile.write('{}\n'.format(target))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write(xml.sax.saxutils.escape(xmldata))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('>\n')
cdashfile.write('{}\n'.format(enddatetime.strftime("%b %d %H:%M %Z")))
cdashfile.write('{}\n'.format(endtime))
cdashfile.write('{}\n'.format((enddatetime - startdatetime).total_seconds() / 60.0))
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.write('\n')
cdashfile.close()
# Create your header as required
#headers = {"content-type": "application/xml"}
print('upload test')
with open('Test.xml', 'r') as fh:
try:
mydata = fh.read().encode('utf-8')
except Exception as e:
print('Read File Exception: = {}'.format(e))
try:
resp = requests.put(url, data=mydata, params=payload) #, headers=headers)
print('url={} resp.status_code={}'.format(url, resp.status_code))
if resp.status_code != 200:
print ('FAILED PUT /builders/build: {}'.format(resp.status_code))
# This means something went wrong.
if resp.status_code != 504: # Gateway timeout
raise Exception('PUT /builders/ {}'.format(resp.status_code))
except Exception as err:
print('Requests Exception: = {}'.format(err))