From 3e0b23d1542325b06bfaa906d0179c13f704aa66 Mon Sep 17 00:00:00 2001 From: "Joel E. Denny" Date: Thu, 10 Oct 2019 19:25:24 +0000 Subject: [PATCH] Revert r374389: "[lit] Clean up internal diff's encoding handling" This breaks a Windows bot. llvm-svn: 374427 --- llvm/utils/lit/lit/builtin_commands/diff.py | 51 +++++++++++-------- .../tests/Inputs/shtest-shell/diff-encodings.txt | 9 ---- .../lit/tests/Inputs/shtest-shell/diff-in.bin | Bin 26 -> 0 bytes .../lit/tests/Inputs/shtest-shell/diff-in.utf16 | Bin 24 -> 0 bytes .../lit/tests/Inputs/shtest-shell/diff-in.utf8 | 3 -- llvm/utils/lit/tests/max-failures.py | 2 +- llvm/utils/lit/tests/shtest-shell.py | 54 +-------------------- 7 files changed, 33 insertions(+), 86 deletions(-) delete mode 100644 llvm/utils/lit/tests/Inputs/shtest-shell/diff-encodings.txt delete mode 100644 llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.bin delete mode 100644 llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf16 delete mode 100644 llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf8 diff --git a/llvm/utils/lit/lit/builtin_commands/diff.py b/llvm/utils/lit/lit/builtin_commands/diff.py index 562b9ac..885b425 100644 --- a/llvm/utils/lit/lit/builtin_commands/diff.py +++ b/llvm/utils/lit/lit/builtin_commands/diff.py @@ -1,7 +1,6 @@ import difflib import functools import getopt -import locale import os import sys @@ -25,26 +24,37 @@ def getDirTree(path, basedir=""): return path, sorted(child_trees) def compareTwoFiles(flags, filepaths): + compare_bytes = False + encoding = None filelines = [] for file in filepaths: - with open(file, 'rb') as file_bin: - filelines.append(file_bin.readlines()) - - try: - return compareTwoTextFiles(flags, filepaths, filelines, - locale.getpreferredencoding(False)) - except UnicodeDecodeError: try: - return compareTwoTextFiles(flags, filepaths, filelines, "utf-8") - except: - return compareTwoBinaryFiles(flags, filepaths, filelines) + with open(file, 'r') as f: + filelines.append(f.readlines()) + except UnicodeDecodeError: + try: + with io.open(file, 'r', encoding="utf-8") as f: + filelines.append(f.readlines()) + encoding = "utf-8" + except: + compare_bytes = True + + if compare_bytes: + return compareTwoBinaryFiles(flags, filepaths) + else: + return compareTwoTextFiles(flags, filepaths, encoding) + +def compareTwoBinaryFiles(flags, filepaths): + filelines = [] + for file in filepaths: + with open(file, 'rb') as f: + filelines.append(f.readlines()) -def compareTwoBinaryFiles(flags, filepaths, filelines): exitCode = 0 if hasattr(difflib, 'diff_bytes'): # python 3.5 or newer diffs = difflib.diff_bytes(difflib.unified_diff, filelines[0], filelines[1], filepaths[0].encode(), filepaths[1].encode()) - diffs = [diff.decode(errors="backslashreplace") for diff in diffs] + diffs = [diff.decode() for diff in diffs] else: # python 2.7 if flags.unified_diff: @@ -58,14 +68,15 @@ def compareTwoBinaryFiles(flags, filepaths, filelines): exitCode = 1 return exitCode -def compareTwoTextFiles(flags, filepaths, filelines_bin, encoding): +def compareTwoTextFiles(flags, filepaths, encoding): filelines = [] - for lines_bin in filelines_bin: - lines = [] - for line_bin in lines_bin: - line = line_bin.decode(encoding=encoding) - lines.append(line) - filelines.append(lines) + for file in filepaths: + if encoding is None: + with open(file, 'r') as f: + filelines.append(f.readlines()) + else: + with io.open(file, 'r', encoding=encoding) as f: + filelines.append(f.readlines()) exitCode = 0 def compose2(f, g): diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/diff-encodings.txt b/llvm/utils/lit/tests/Inputs/shtest-shell/diff-encodings.txt deleted file mode 100644 index d8b9718..0000000 --- a/llvm/utils/lit/tests/Inputs/shtest-shell/diff-encodings.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Check that diff falls back to binary mode if it cannot decode a file. - -# RUN: diff -u diff-in.bin diff-in.bin -# RUN: diff -u diff-in.utf16 diff-in.bin && false || true -# RUN: diff -u diff-in.utf8 diff-in.bin && false || true -# RUN: diff -u diff-in.bin diff-in.utf8 && false || true - -# Fail so lit will print output. -# RUN: false diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.bin b/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.bin deleted file mode 100644 index 06b800b707c1ade254fa995363aa211d096b534e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26 acmZQbW5@?WE`}tAM24dO|DmiZ1}*?iD+SX4 diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf16 b/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf16 deleted file mode 100644 index d7d9feefa7da8341244deff11b6ea91b4d36b8f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 YcmZQbW5@?WE`}tAM1~?LUB$o!05LiP)c^nh diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf8 b/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf8 deleted file mode 100644 index 86e041d..0000000 --- a/llvm/utils/lit/tests/Inputs/shtest-shell/diff-in.utf8 +++ /dev/null @@ -1,3 +0,0 @@ -foo -bar -baz diff --git a/llvm/utils/lit/tests/max-failures.py b/llvm/utils/lit/tests/max-failures.py index 5149a91..cee06fa 100644 --- a/llvm/utils/lit/tests/max-failures.py +++ b/llvm/utils/lit/tests/max-failures.py @@ -8,7 +8,7 @@ # # END. -# CHECK: Failing Tests (28) +# CHECK: Failing Tests (27) # CHECK: Failing Tests (1) # CHECK: Failing Tests (2) # CHECK: error: argument --max-failures: requires positive integer, but found '0' diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py index 6d9b1aa..3978e44 100644 --- a/llvm/utils/lit/tests/shtest-shell.py +++ b/llvm/utils/lit/tests/shtest-shell.py @@ -34,58 +34,6 @@ # CHECK: error: command failed with exit status: 127 # CHECK: *** - -# CHECK: FAIL: shtest-shell :: diff-encodings.txt -# CHECK: *** TEST 'shtest-shell :: diff-encodings.txt' FAILED *** - -# CHECK: $ "diff" "-u" "diff-in.bin" "diff-in.bin" -# CHECK-NOT: error - -# CHECK: $ "diff" "-u" "diff-in.utf16" "diff-in.bin" -# CHECK: # command output: -# CHECK-NEXT: --- -# CHECK-NEXT: +++ -# CHECK-NEXT: @@ -# CHECK-NEXT: {{^ .f.o.o.$}} -# CHECK-NEXT: {{^-.b.a.r.$}} -# CHECK-NEXT: {{^\+.b.a.r..}} -# CHECK-NEXT: {{^ .b.a.z.$}} -# CHECK: error: command failed with exit status: 1 -# CHECK: $ "true" - -# CHECK: $ "diff" "-u" "diff-in.utf8" "diff-in.bin" -# CHECK: # command output: -# CHECK-NEXT: --- -# CHECK-NEXT: +++ -# CHECK-NEXT: @@ -# CHECK-NEXT: -foo -# CHECK-NEXT: -bar -# CHECK-NEXT: -baz -# CHECK-NEXT: {{^\+.f.o.o.$}} -# CHECK-NEXT: {{^\+.b.a.r..}} -# CHECK-NEXT: {{^\+.b.a.z.$}} -# CHECK: error: command failed with exit status: 1 -# CHECK: $ "true" - -# CHECK: $ "diff" "-u" "diff-in.bin" "diff-in.utf8" -# CHECK: # command output: -# CHECK-NEXT: --- -# CHECK-NEXT: +++ -# CHECK-NEXT: @@ -# CHECK-NEXT: {{^\-.f.o.o.$}} -# CHECK-NEXT: {{^\-.b.a.r..}} -# CHECK-NEXT: {{^\-.b.a.z.$}} -# CHECK-NEXT: +foo -# CHECK-NEXT: +bar -# CHECK-NEXT: +baz -# CHECK: error: command failed with exit status: 1 -# CHECK: $ "true" - -# CHECK: $ "false" - -# CHECK: *** - - # CHECK: FAIL: shtest-shell :: diff-error-1.txt # CHECK: *** TEST 'shtest-shell :: diff-error-1.txt' FAILED *** # CHECK: $ "diff" "-B" "temp1.txt" "temp2.txt" @@ -297,4 +245,4 @@ # CHECK: PASS: shtest-shell :: sequencing-0.txt # CHECK: XFAIL: shtest-shell :: sequencing-1.txt # CHECK: PASS: shtest-shell :: valid-shell.txt -# CHECK: Failing Tests (28) +# CHECK: Failing Tests (27) -- 2.7.4