Skip to content

Implement --substitute #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PapyChacal wants to merge 12 commits into
base: main
Choose a base branch
from
Open

Implement --substitute #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
5b22ace
light refactor.
PapyChacal Jul 30, 2024
567020f
Bump filecheck.
PapyChacal Jul 30, 2024
7db5a1f
Factor out conditions in creating the right anonymization function.
PapyChacal Jul 30, 2024
ec714e4
Add xDSL/MLIR substitution.
PapyChacal Jul 30, 2024
0bb823d
Add file split scoping for substituion
PapyChacal Jul 30, 2024
e65979e
EOL
PapyChacal Jul 30, 2024
f40d0f2
ruff
PapyChacal Jul 30, 2024
60696ef
Pyright.
PapyChacal Jul 30, 2024
2a03a37
Pyright 3.7?
PapyChacal Jul 30, 2024
33e6543
.
PapyChacal Jul 30, 2024
c48f3b3
Just bye 3.7.
PapyChacal Jul 30, 2024
74c700a
Literal filecheck.
PapyChacal Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci-pyright-fails.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

env:
PYRIGHT_VERSION: 1.0
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/code-formatting.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
python-version: ['3.8', '3.9', '3.10', '3.11']

steps:
- uses: actions/checkout@v3
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
python-version: ['3.8', '3.9', '3.10', '3.11']

steps:
- uses: actions/checkout@v3
Expand Down
117 changes: 101 additions & 16 deletions filecheckize/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,30 @@
from io import TextIOWrapper
from typing import cast

UNNAMED_SSA_VALUE = re.compile(r"%([\d]+)")
SSA_VALUE_NAME = re.compile(r"%([\d]+|[\w$._-][\w\d$._-]*)(:|#[\d]*)?")
BASIC_BLOCK_NAME = re.compile(r"\^([\d]+|[\w$._-][\w\d$._-]*)")

ANONYMOUS_VARIABLE = r"%{{.*}}"
ANONYMOUS_BLOCK = r"^{{.*}}"

SUBSTITUTED_VARS: "set[str]" = set()
SUBSTITUTED_BLOCKS: "set[str]" = set()


def substitute_variable(var: str) -> str:
if var not in SUBSTITUTED_VARS:
SUBSTITUTED_VARS.add(var)
return f"[[v{var}:%.*]]"
return f"[[v{var}]]"


def substitute_block(block: str) -> str:
if block not in SUBSTITUTED_BLOCKS:
SUBSTITUTED_BLOCKS.add(block)
return f"[[b{block}:^.*]]"
return f"[[b{block}]]"


def main():
parser = argparse.ArgumentParser(
Expand All @@ -29,6 +53,11 @@ def main():
action="store_true",
help="Anonymize MLIR unnamed SSA values and basic block names.",
)
parser.add_argument(
"--substitute",
action="store_true",
help="Use variable substituion instead of anonymization.",
)
parser.add_argument(
"--check-empty-lines",
action="store_true",
Expand All @@ -55,9 +84,71 @@ def main():
args = parser.parse_args(sys.argv[1:])

comment_line = re.compile(rf"^\s*{re.escape(args.comments_prefix)}.*$")
unnamed_ssa_value = re.compile(r"%([\d]+)")
ssa_value_name = re.compile(r"%([\d]+|[\w$._-][\w\d$._-]*)(:|#[\d]*)?")
basic_block_name = re.compile(r"\^([\d]+|[\w$._-][\w\d$._-]*)")

if args.strip_comments:

def strip(line: str) -> bool:
return bool(re.match(comment_line, line))

else:

def strip(line: str) -> bool:
return False

if args.mlir_anonymize:

if args.substitute:

def anonymize(line: str) -> str:
# Anonymize SSA value names
return re.sub(
BASIC_BLOCK_NAME,
lambda m: substitute_block(m.group(1)),
re.sub(
SSA_VALUE_NAME, lambda m: substitute_variable(m.group(1)), line
),
)

else:

def anonymize(line: str) -> str:
# Anonymize SSA value names
return re.sub(
BASIC_BLOCK_NAME,
ANONYMOUS_BLOCK,
re.sub(SSA_VALUE_NAME, ANONYMOUS_VARIABLE, line),
)

elif args.xdsl_anonymize:

if args.substitute:

def anonymize(line: str) -> str:
# Anonymize SSA value names
return re.sub(
BASIC_BLOCK_NAME,
lambda m: substitute_block(m.group(1)),
re.sub(
UNNAMED_SSA_VALUE,
lambda m: substitute_variable(m.group(1)),
line,
),
)

else:

def anonymize(line: str) -> str:
# Anonymize SSA value names
return re.sub(
BASIC_BLOCK_NAME,
ANONYMOUS_BLOCK,
re.sub(UNNAMED_SSA_VALUE, ANONYMOUS_VARIABLE, line),
)

else:

def anonymize(line: str) -> str:
return line

prefix = args.check_prefix

Expand All @@ -75,21 +166,15 @@ def main():
next = False
continue

# Ignore remaining comment lines
if args.strip_comments:
if re.match(comment_line, line):
continue
elif "// -----" in line:
SUBSTITUTED_VARS.clear()
SUBSTITUTED_BLOCKS.clear()

if args.mlir_anonymize or args.xdsl_anonymize:
if args.mlir_anonymize:
# Anonymize SSA value names
line = re.sub(ssa_value_name, r"%{{.*}}", line)
elif args.xdsl_anonymize:
# Anonymize unnamed SSA values
line = re.sub(unnamed_ssa_value, r"%{{.*}}", line)
# Ignore remaining comment lines
if strip(line):
continue

# Anonymize basic blocks names
line = re.sub(basic_block_name, r"^{{.*}}", line)
line = anonymize(line)

# Print the modified line
if next:
Expand Down
2 changes: 1 addition & 1 deletion requirements-optional.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
black<24.5
toml<0.11
coverage
filecheck<0.025
filecheck<=1.0
lit<18.2
ruff<0.5
isort<5.14
Expand Down
86 changes: 52 additions & 34 deletions tests/filecheck/mlir.mlir
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
// RUN: filecheckize %s --strip-comments | filecheck %s --match-full-lines --check-prefix STRIP
// RUN: filecheckize %s --strip-comments --check-empty-lines | filecheck %s --check-prefix WITH-EMPTY --match-full-lines
// RUN: filecheckize %s --strip-comments --mlir-anonymize | filecheck %s --check-prefix ANONYMIZE --match-full-lines
// RUN: filecheckize %s --strip-comments --mlir-anonymize | filecheck %s --check-prefix MLIR-ANONYMIZE --match-full-lines
// RUN: filecheckize %s --strip-comments --mlir-anonymize --substitute | filecheck %s --check-prefix MLIR-SUBSTITUTE --match-full-lines
// RUN: filecheckize %s --strip-comments --xdsl-anonymize | filecheck %s --check-prefix XDSL-ANONYMIZE --match-full-lines
// RUN: filecheckize %s --strip-comments --xdsl-anonymize --substitute | filecheck %s --check-prefix XDSL-SUBSTITUTE --match-full-lines

func.func @arg_rec(%0 : !test.type<"int">) -> !test.type<"int"> {
%1 = func.call @arg_rec(%0) : (!test.type<"int">) -> !test.type<"int">
Expand All @@ -11,39 +13,55 @@ func.func @arg_rec(%0 : !test.type<"int">) -> !test.type<"int"> {
func.return %1 : !test.type<"int">
}

// STRIP: // CHECK: func.func @arg_rec(%0 : !test.type<"int">) -> !test.type<"int"> {
// STRIP-NEXT: // CHECK-NEXT: %1 = func.call @arg_rec(%0) : (!test.type<"int">) -> !test.type<"int">
// STRIP-NEXT: // CHECK-NEXT: %name = arith.constant : i32
// STRIP-NEXT: // CHECK-NEXT: %other_name = arith.constant : i32
// STRIP-NEXT: // CHECK-NEXT: %2 = arith.addi %name, %other_name : i32
// STRIP-NEXT: // CHECK-NEXT: func.return %1 : !test.type<"int">
// STRIP-NEXT: // CHECK-NEXT: }
// STRIP{LITERAL}: // CHECK: func.func @arg_rec(%0 : !test.type<"int">) -> !test.type<"int"> {
// STRIP-NEXT{LITERAL}: // CHECK-NEXT: %1 = func.call @arg_rec(%0) : (!test.type<"int">) -> !test.type<"int">
// STRIP-NEXT{LITERAL}: // CHECK-NEXT: %name = arith.constant : i32
// STRIP-NEXT{LITERAL}: // CHECK-NEXT: %other_name = arith.constant : i32
// STRIP-NEXT{LITERAL}: // CHECK-NEXT: %2 = arith.addi %name, %other_name : i32
// STRIP-NEXT{LITERAL}: // CHECK-NEXT: func.return %1 : !test.type<"int">
// STRIP-NEXT{LITERAL}: // CHECK-NEXT: }

// WITH-EMPTY: // CHECK-EMPTY:
// WITH-EMPTY-NEXT: // CHECK-NEXT: func.func @arg_rec(%0 : !test.type<"int">) -> !test.type<"int"> {
// WITH-EMPTY-NEXT: // CHECK-NEXT: %1 = func.call @arg_rec(%0) : (!test.type<"int">) -> !test.type<"int">
// WITH-EMPTY-NEXT: // CHECK-NEXT: %name = arith.constant : i32
// WITH-EMPTY-NEXT: // CHECK-NEXT: %other_name = arith.constant : i32
// WITH-EMPTY-NEXT: // CHECK-NEXT: %2 = arith.addi %name, %other_name : i32
// WITH-EMPTY-NEXT: // CHECK-NEXT: func.return %1 : !test.type<"int">
// WITH-EMPTY-NEXT: // CHECK-NEXT: }
// WITH-EMPTY-NEXT: // CHECK-EMPTY:
// WITH-EMPTY-NEXT: // CHECK-EMPTY:
// WITH-EMPTY-NEXT: // CHECK-EMPTY:
// WITH-EMPTY-NEXT: // CHECK-EMPTY:
// WITH-EMPTY{LITERAL}: // CHECK-EMPTY:
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: func.func @arg_rec(%0 : !test.type<"int">) -> !test.type<"int"> {
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: %1 = func.call @arg_rec(%0) : (!test.type<"int">) -> !test.type<"int">
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: %name = arith.constant : i32
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: %other_name = arith.constant : i32
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: %2 = arith.addi %name, %other_name : i32
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: func.return %1 : !test.type<"int">
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-NEXT: }
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-EMPTY:
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-EMPTY:
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-EMPTY:
// WITH-EMPTY-NEXT{LITERAL}: // CHECK-EMPTY:

// ANONYMIZE: // CHECK: func.func @arg_rec(%{{.*}} : !test.type<"int">) -> !test.type<"int"> {
// ANONYMIZE-NEXT: // CHECK-NEXT: %{{.*}} = func.call @arg_rec(%{{.*}}) : (!test.type<"int">) -> !test.type<"int">
// ANONYMIZE-NEXT: // CHECK-NEXT: %{{.*}} = arith.constant : i32
// ANONYMIZE-NEXT: // CHECK-NEXT: %{{.*}} = arith.constant : i32
// ANONYMIZE-NEXT: // CHECK-NEXT: %{{.*}} = arith.addi %{{.*}}, %{{.*}} : i32
// ANONYMIZE-NEXT: // CHECK-NEXT: func.return %{{.*}} : !test.type<"int">
// ANONYMIZE-NEXT: // CHECK-NEXT: }
// MLIR-ANONYMIZE{LITERAL}: // CHECK: func.func @arg_rec(%{{.*}} : !test.type<"int">) -> !test.type<"int"> {
// MLIR-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %{{.*}} = func.call @arg_rec(%{{.*}}) : (!test.type<"int">) -> !test.type<"int">
// MLIR-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %{{.*}} = arith.constant : i32
// MLIR-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %{{.*}} = arith.constant : i32
// MLIR-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %{{.*}} = arith.addi %{{.*}}, %{{.*}} : i32
// MLIR-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: func.return %{{.*}} : !test.type<"int">
// MLIR-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: }

// XDSL-ANONYMIZE: // CHECK: func.func @arg_rec(%{{.*}} : !test.type<"int">) -> !test.type<"int"> {
// XDSL-ANONYMIZE-NEXT: // CHECK-NEXT: %{{.*}} = func.call @arg_rec(%{{.*}}) : (!test.type<"int">) -> !test.type<"int">
// XDSL-ANONYMIZE-NEXT: // CHECK-NEXT: %name = arith.constant : i32
// XDSL-ANONYMIZE-NEXT: // CHECK-NEXT: %other_name = arith.constant : i32
// XDSL-ANONYMIZE-NEXT: // CHECK-NEXT: %{{.*}} = arith.addi %name, %other_name : i32
// XDSL-ANONYMIZE-NEXT: // CHECK-NEXT: func.return %{{.*}} : !test.type<"int">
// XDSL-ANONYMIZE-NEXT: // CHECK-NEXT: }
// MLIR-SUBSTITUTE{LITERAL}: // CHECK: func.func @arg_rec([[v0:%.*]] : !test.type<"int">) -> !test.type<"int"> {
// MLIR-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: [[v1:%.*]] = func.call @arg_rec([[v0]]) : (!test.type<"int">) -> !test.type<"int">
// MLIR-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: [[vname:%.*]] = arith.constant : i32
// MLIR-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: [[vother_name:%.*]] = arith.constant : i32
// MLIR-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: [[v2:%.*]] = arith.addi [[vname]], [[vother_name]] : i32
// MLIR-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: func.return [[v1]] : !test.type<"int">
// MLIR-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: }

// XDSL-ANONYMIZE{LITERAL}: // CHECK: func.func @arg_rec(%{{.*}} : !test.type<"int">) -> !test.type<"int"> {
// XDSL-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %{{.*}} = func.call @arg_rec(%{{.*}}) : (!test.type<"int">) -> !test.type<"int">
// XDSL-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %name = arith.constant : i32
// XDSL-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %other_name = arith.constant : i32
// XDSL-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: %{{.*}} = arith.addi %name, %other_name : i32
// XDSL-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: func.return %{{.*}} : !test.type<"int">
// XDSL-ANONYMIZE-NEXT{LITERAL}: // CHECK-NEXT: }

// XDSL-SUBSTITUTE{LITERAL}: // CHECK: func.func @arg_rec([[v0:%.*]] : !test.type<"int">) -> !test.type<"int"> {
// XDSL-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: [[v1:%.*]] = func.call @arg_rec([[v0]]) : (!test.type<"int">) -> !test.type<"int">
// XDSL-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: %name = arith.constant : i32
// XDSL-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: %other_name = arith.constant : i32
// XDSL-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: [[v2:%.*]] = arith.addi %name, %other_name : i32
// XDSL-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: func.return [[v1]] : !test.type<"int">
// XDSL-SUBSTITUTE-NEXT{LITERAL}: // CHECK-NEXT: }