import os import subprocess import sys import json from itertools import product def tail_lines(s, n, f=3): ss = s.split('\n') if len(ss) > n*f: return '\n'.join(["... {} lines ...".format(len(ss)-n)] + ss[-n:]) return s def run_fw(config_filepath, env): cmd = './framework.exe {} -m pre'.format(config_filepath) try: print("Running Framework...") cp = subprocess.run(cmd, capture_output=True, check=True, shell=True, text=True, env=env) print("__STDOUT__\n" + tail_lines(cp.stdout, 10)) #print("__STDERR__\n" + tail_lines(cp.stderr, 5)) except subprocess.CalledProcessError as e: print("__STDOUT__\n" + tail_lines(e.stdout, 10)) print("__STDERR__\n" + tail_lines(e.stderr, 10)) print("Error in Framework :-/\n") return False else: print("Success!\n") return True # This is for convenience in writing environment JSON files, # so that length-one arrays can just be the one null/string value. inline_list = lambda s: s if isinstance(s, list) else [s] def iter_space(space): var, span = space.keys(), space.values() span = map(inline_list, span) # span is now a list of lists # Iterate over every combination for _v in product(*span): # Construct the environment dictionary and return it yield dict(zip(var, _v)) def main(config_filepath, env_space): n_success = 0 total = 0 for _env in iter_space(env_space): #env = os.environ.copy() # Not recommended env = dict() for k, v in _env.items(): if v is None: if k in env: del env[k] else: print(">>> {}={}".format(k, v or '')) env[k] = v ret = run_fw(config_filepath, env) n_success += ret total += 1 print("{} out of {} runs were successful".format(n_success, total)) if __name__ == '__main__': if len(sys.argv) < 3: print("Usage: python configuration_test.py config.xml env.json") sys.exit() config_filepath = sys.argv[1] with open(sys.argv[2], 'r') as f: env_space = json.load(f) main(config_filepath, env_space)