From cd3963f117b6d523efbf5f02f9ea800582a26d99 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 13:47:25 +0530 Subject: [PATCH] fix: respect DOCKEREXE configuration and remove hardcoded sudo docker (fixes #343) --- mkconcore.py | 282 +++++++++++++++++++++++++-------------------------- 1 file changed, 141 insertions(+), 141 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 0a11948..3f8bba8 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -75,46 +75,46 @@ import shlex # Added for POSIX shell escaping # input validation helper -def safe_name(value, context, allow_path=False): - """ - Validates that the input string does not contain characters dangerous - for filesystem paths or shell command injection. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - # blocks control characters and shell metacharacters - # allow path separators and drive colons for full paths when needed - if allow_path: - pattern = r'[\x00-\x1F\x7F*?"<>|;&`$\'()]' - else: - # blocks path traversal (/, \, :) in addition to shell metacharacters - pattern = r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]' - if re.search(pattern, value): - raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") - return value - -def safe_relpath(value, context): - """ - Allow relative subpaths while blocking traversal and absolute/drive paths. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - normalized = value.replace("\\", "/") - safe_name(normalized, context, allow_path=True) - if normalized.startswith("/") or normalized.startswith("~"): - raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") - if re.match(r"^[A-Za-z]:", normalized): - raise ValueError(f"Unsafe {context}: drive paths are not allowed.") - if ":" in normalized: - raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") - if any(part in ("", "..") for part in normalized.split("/")): - raise ValueError(f"Unsafe {context}: invalid path segment.") - return normalized - -MKCONCORE_VER = "22-09-18" - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - +def safe_name(value, context, allow_path=False): + """ + Validates that the input string does not contain characters dangerous + for filesystem paths or shell command injection. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # blocks control characters and shell metacharacters + # allow path separators and drive colons for full paths when needed + if allow_path: + pattern = r'[\x00-\x1F\x7F*?"<>|;&`$\'()]' + else: + # blocks path traversal (/, \, :) in addition to shell metacharacters + pattern = r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]' + if re.search(pattern, value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value + +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_name(normalized, context, allow_path=True) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized + +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + def _load_tool_config(filepath): tools = {} with open(filepath, "r") as f: @@ -128,23 +128,23 @@ def _load_tool_config(filepath): tools[k] = v return tools -def _resolve_concore_path(): - script_concore = os.path.join(SCRIPT_DIR, "concore.py") - if os.path.exists(script_concore): - return SCRIPT_DIR - cwd_concore = os.path.join(os.getcwd(), "concore.py") - if os.path.exists(cwd_concore): - return os.getcwd() - return SCRIPT_DIR - -if len(sys.argv) < 4: - print("usage: py mkconcore.py file.graphml sourcedir outdir [type]") - print(" type must be posix (macos or ubuntu), windows, or docker") - sys.exit(1) - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = _resolve_concore_path() +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +if len(sys.argv) < 4: + print("usage: py mkconcore.py file.graphml sourcedir outdir [type]") + print(" type must be posix (macos or ubuntu), windows, or docker") + sys.exit(1) + +GRAPHML_FILE = sys.argv[1] +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() CPPWIN = os.environ.get("CONCORE_CPPWIN", "g++") #Windows C++ 6/22/21 CPPEXE = os.environ.get("CONCORE_CPPEXE", "g++") #Ubuntu/macOS C++ 6/22/21 VWIN = os.environ.get("CONCORE_VWIN", "iverilog") #Windows verilog 6/25/21 @@ -157,7 +157,7 @@ def _resolve_concore_path(): OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime -DOCKEREXE = "sudo docker"#assume simple docker install +DOCKEREXE = os.environ.get("DOCKEREXE", "docker")#default to docker, allow env override DOCKEREPO = "markgarnold"#where pulls come from 3/28/21 INDIRNAME = ":/in" OUTDIRNAME = ":/out" @@ -200,8 +200,8 @@ def _resolve_concore_path(): sourcedir = sys.argv[2] outdir = sys.argv[3] -# Validate outdir argument (allow full paths) -safe_name(outdir, "Output directory argument", allow_path=True) +# Validate outdir argument (allow full paths) +safe_name(outdir, "Output directory argument", allow_path=True) if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") @@ -292,15 +292,15 @@ def cleanup_script_files(): node_label = re.sub(r'(\s+|\n)', ' ', node_label) #Validate node labels - if ':' in node_label: - container_part, source_part = node_label.split(':', 1) - safe_name(container_part, f"Node container name '{container_part}'") - source_part = safe_relpath(source_part, f"Node source file '{source_part}'") - node_label = f"{container_part}:{source_part}" - else: - safe_name(node_label, f"Node label '{node_label}'") - # Explicitly reject incorrect format to prevent later crashes and ambiguity - raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") + if ':' in node_label: + container_part, source_part = node_label.split(':', 1) + safe_name(container_part, f"Node container name '{container_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" + else: + safe_name(node_label, f"Node label '{node_label}'") + # Explicitly reject incorrect format to prevent later crashes and ambiguity + raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] @@ -486,15 +486,15 @@ def cleanup_script_files(): if not sourcecode: continue - if "." in sourcecode: - dockername, langext = os.path.splitext(sourcecode) - else: - dockername, langext = sourcecode, "" - - script_target_path = os.path.join(outdir, "src", sourcecode) - script_target_parent = os.path.dirname(script_target_path) - if script_target_parent: - os.makedirs(script_target_parent, exist_ok=True) + if "." in sourcecode: + dockername, langext = os.path.splitext(sourcecode) + else: + dockername, langext = sourcecode, "" + + script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: @@ -684,33 +684,33 @@ def cleanup_script_files(): # 4. Write final iport/oport files logging.info("Writing .iport and .oport files...") -for node_label, ports in node_port_mappings.items(): +for node_label, ports in node_port_mappings.items(): try: containername, sourcecode = node_label.split(':', 1) - if not sourcecode or "." not in sourcecode: continue - dockername = os.path.splitext(sourcecode)[0] - iport_path = os.path.join(outdir, "src", f"{dockername}.iport") - oport_path = os.path.join(outdir, "src", f"{dockername}.oport") - iport_parent = os.path.dirname(iport_path) - if iport_parent: - os.makedirs(iport_parent, exist_ok=True) - with open(iport_path, "w") as fport: - fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) - with open(oport_path, "w") as fport: - fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) + if not sourcecode or "." not in sourcecode: continue + dockername = os.path.splitext(sourcecode)[0] + iport_path = os.path.join(outdir, "src", f"{dockername}.iport") + oport_path = os.path.join(outdir, "src", f"{dockername}.oport") + iport_parent = os.path.dirname(iport_path) + if iport_parent: + os.makedirs(iport_parent, exist_ok=True) + with open(iport_path, "w") as fport: + fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) + with open(oport_path, "w") as fport: + fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) except ValueError: continue #if docker, make docker-dirs, generate build, run, stop, clear scripts and quit -if (concoretype=="docker"): - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.rsplit(".", 1) - dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") - if not os.path.exists(dockerfile_path): # 3/30/21 - try: +if (concoretype=="docker"): + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.rsplit(".", 1) + dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") + if not os.path.exists(dockerfile_path): # 3/30/21 + try: if langext=="py": src_path = CONCOREPATH+"/Dockerfile.py" logging.info("assuming .py extension for Dockerfile") @@ -728,14 +728,14 @@ def cleanup_script_files(): logging.info("assuming .m extension for Dockerfile") with open(src_path) as fsource: source_content = fsource.read() - except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() - dockerfile_parent = os.path.dirname(dockerfile_path) - if dockerfile_parent: - os.makedirs(dockerfile_parent, exist_ok=True) - with open(dockerfile_path,"w") as fcopy: - fcopy.write(source_content) + except: + logging.error(f"{CONCOREPATH} is not correct path to concore") + quit() + dockerfile_parent = os.path.dirname(dockerfile_path) + if dockerfile_parent: + os.makedirs(dockerfile_parent, exist_ok=True) + with open(dockerfile_path,"w") as fcopy: + fcopy.write(source_content) if langext=="py": fcopy.write('CMD ["python", "-i", "'+sourcecode+'"]\n') if langext=="m": @@ -827,8 +827,8 @@ def cleanup_script_files(): fmaxtime.write('#!/bin/bash' + "\n") fmaxtime.write('echo "$1" >concore.maxtime\n') fmaxtime.write('echo "FROM alpine:3.8" > Dockerfile\n') - fmaxtime.write('sudo docker build -t docker-concore .\n') - fmaxtime.write('sudo docker run --name=concore') + fmaxtime.write(f'{DOCKEREXE} build -t docker-concore .\n') + fmaxtime.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: @@ -854,15 +854,15 @@ def cleanup_script_files(): dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fmaxtime.write('sudo docker cp concore.maxtime concore:/') + fmaxtime.write(f'{DOCKEREXE} cp concore.maxtime concore:/') # escape destination path in docker cp vol_path = writeedges.split(":")[0].split("-v ")[1].strip() fmaxtime.write(shlex.quote(vol_path+"/concore.maxtime")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - fmaxtime.write('sudo docker stop concore \n') - fmaxtime.write('sudo docker rm concore\n') - fmaxtime.write('sudo docker rmi docker-concore\n') + fmaxtime.write(f'{DOCKEREXE} stop concore \n') + fmaxtime.write(f'{DOCKEREXE} rm concore\n') + fmaxtime.write(f'{DOCKEREXE} rmi docker-concore\n') fmaxtime.write('rm Dockerfile\n') fmaxtime.write('rm concore.maxtime\n') fmaxtime.close() @@ -870,8 +870,8 @@ def cleanup_script_files(): fparams.write('#!/bin/bash' + "\n") fparams.write('echo "$1" >concore.params\n') fparams.write('echo "FROM alpine:3.8" > Dockerfile\n') - fparams.write('sudo docker build -t docker-concore .\n') - fparams.write('sudo docker run --name=concore') + fparams.write(f'{DOCKEREXE} build -t docker-concore .\n') + fparams.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: @@ -897,23 +897,23 @@ def cleanup_script_files(): dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fparams.write('sudo docker cp concore.params concore:/') + fparams.write(f'{DOCKEREXE} cp concore.params concore:/') # escape destination path vol_path = writeedges.split(":")[0].split("-v ")[1].strip() fparams.write(shlex.quote(vol_path+"/concore.params")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - fparams.write('sudo docker stop concore \n') - fparams.write('sudo docker rm concore\n') - fparams.write('sudo docker rmi docker-concore\n') + fparams.write(f'{DOCKEREXE} stop concore \n') + fparams.write(f'{DOCKEREXE} rm concore\n') + fparams.write(f'{DOCKEREXE} rmi docker-concore\n') fparams.write('rm Dockerfile\n') fparams.write('rm concore.params\n') fparams.close() funlock.write('#!/bin/bash' + "\n") funlock.write('echo "FROM alpine:3.8" > Dockerfile\n') - funlock.write('sudo docker build -t docker-concore .\n') - funlock.write('sudo docker run --name=concore') + funlock.write(f'{DOCKEREXE} build -t docker-concore .\n') + funlock.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: @@ -939,15 +939,15 @@ def cleanup_script_files(): dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - funlock.write('sudo docker cp ~/concore.apikey concore:/') + funlock.write(f'{DOCKEREXE} cp ~/concore.apikey concore:/') # escape destination path vol_path = writeedges.split(":")[0].split("-v ")[1].strip() funlock.write(shlex.quote(vol_path+"/concore.apikey")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - funlock.write('sudo docker stop concore \n') - funlock.write('sudo docker rm concore\n') - funlock.write('sudo docker rmi docker-concore\n') + funlock.write(f'{DOCKEREXE} stop concore \n') + funlock.write(f'{DOCKEREXE} rm concore\n') + funlock.write(f'{DOCKEREXE} rmi docker-concore\n') funlock.write('rm Dockerfile\n') funlock.close() @@ -979,22 +979,22 @@ def cleanup_script_files(): if concoretype=="posix": fbuild.write('#!/bin/bash' + "\n") -for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0: - if sourcecode.find(".")==-1: - logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 - quit() - dockername,langext = sourcecode.rsplit(".", 1) - fbuild.write('mkdir '+containername+"\n") - source_subdir = os.path.dirname(sourcecode).replace("\\", "/") - if source_subdir: - if concoretype == "windows": - fbuild.write("mkdir .\\"+containername+"\\"+source_subdir.replace("/", "\\")+"\n") - else: - fbuild.write("mkdir -p ./"+containername+"/"+source_subdir+"\n") - if concoretype == "windows": - fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") +for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0: + if sourcecode.find(".")==-1: + logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 + quit() + dockername,langext = sourcecode.rsplit(".", 1) + fbuild.write('mkdir '+containername+"\n") + source_subdir = os.path.dirname(sourcecode).replace("\\", "/") + if source_subdir: + if concoretype == "windows": + fbuild.write("mkdir .\\"+containername+"\\"+source_subdir.replace("/", "\\")+"\n") + else: + fbuild.write("mkdir -p ./"+containername+"/"+source_subdir+"\n") + if concoretype == "windows": + fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") if langext == "py": fbuild.write("copy .\\src\\concore.py .\\" + containername + "\\concore.py\n") elif langext == "cpp": @@ -1290,4 +1290,4 @@ def cleanup_script_files(): os.chmod(outdir+"/clear",stat.S_IRWXU) os.chmod(outdir+"/maxtime",stat.S_IRWXU) os.chmod(outdir+"/params",stat.S_IRWXU) - os.chmod(outdir+"/unlock",stat.S_IRWXU) + os.chmod(outdir+"/unlock",stat.S_IRWXU)