[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v2 14/17] iotests/297: split linters.py off from 297
From: |
John Snow |
Subject: |
[PATCH v2 14/17] iotests/297: split linters.py off from 297 |
Date: |
Tue, 20 Jul 2021 13:33:33 -0400 |
Split the linter execution itself out from 297, leaving just environment
setup in 297. This is done so that non-iotest code can invoke the
linters without needing to worry about imports of unpackaged iotest
code.
Eventually, it should be possible to replace linters.py with mypy.ini
and pylintrc files that instruct these tools how to run properly in this
directory, but ... not yet, and not in this series.
Signed-off-by: John Snow <jsnow@redhat.com>
---
tests/qemu-iotests/297 | 110 ++-----------------------------
tests/qemu-iotests/linters.py | 121 ++++++++++++++++++++++++++++++++++
2 files changed, 125 insertions(+), 106 deletions(-)
create mode 100755 tests/qemu-iotests/linters.py
diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297
index ca9ee72d2fa..3d29af5b78a 100755
--- a/tests/qemu-iotests/297
+++ b/tests/qemu-iotests/297
@@ -17,115 +17,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
-import re
import shutil
-import subprocess
-import sys
-from typing import List, Mapping, Optional
import iotests
+import linters
-# TODO: Empty this list!
-SKIP_FILES = (
- '030', '040', '041', '044', '045', '055', '056', '057', '065', '093',
- '096', '118', '124', '132', '136', '139', '147', '148', '149',
- '151', '152', '155', '163', '165', '169', '194', '196', '199', '202',
- '203', '205', '206', '207', '208', '210', '211', '212', '213', '216',
- '218', '219', '222', '224', '228', '234', '235', '236', '237', '238',
- '240', '242', '245', '246', '248', '255', '256', '257', '258', '260',
- '262', '264', '266', '274', '277', '280', '281', '295', '296', '298',
- '299', '302', '303', '304', '307',
- 'nbd-fault-injector.py', 'qcow2.py', 'qcow2_format.py', 'qed.py'
-)
-
-
-def is_python_file(filepath: str) -> bool:
- if filepath.endswith('.py'):
- return True
-
- with open(filepath) as f:
- try:
- first_line = f.readline()
- return re.match('^#!.*python', first_line) is not None
- except UnicodeDecodeError: # Ignore binary files
- return False
-
-
-def get_test_files(directory: str = '.') -> List[str]:
- files = []
-
- iotests.logger.debug("get_test_files(%s)", directory)
- for dirent in os.scandir(directory):
- if dirent.name in SKIP_FILES:
- continue
-
- relpath = os.path.join(directory, dirent.name)
- if dirent.is_dir():
- files.extend(get_test_files(relpath))
- elif is_python_file(relpath):
- files.append(relpath)
-
- return files
-
-
-def run_linters(
- files: List[str],
- directory: str = '.',
- env: Optional[Mapping[str, str]] = None,
-) -> int:
- ret = 0
-
- print('=== pylint ===')
- sys.stdout.flush()
-
- # Todo notes are fine, but fixme's or xxx's should probably just be
- # fixed (in tests, at least)
- p = subprocess.run(
- ('python3', '-m', 'pylint', '--score=n', '--notes=FIXME,XXX', *files),
- cwd=directory,
- env=env,
- check=False,
- universal_newlines=True,
- )
- ret += p.returncode
-
- print('=== mypy ===')
- sys.stdout.flush()
-
- # We have to call mypy separately for each file. Otherwise, it
- # will interpret all given files as belonging together (i.e., they
- # may not both define the same classes, etc.; most notably, they
- # must not both define the __main__ module).
- for filename in files:
- p = subprocess.run(
- (
- 'python3', '-m', 'mypy',
- '--warn-unused-configs',
- '--disallow-subclassing-any',
- '--disallow-any-generics',
- '--disallow-incomplete-defs',
- '--disallow-untyped-decorators',
- '--no-implicit-optional',
- '--warn-redundant-casts',
- '--warn-unused-ignores',
- '--no-implicit-reexport',
- '--namespace-packages',
- filename,
- ),
- cwd=directory,
- env=env,
- check=False,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- universal_newlines=True
- )
-
- ret += p.returncode
- if p.returncode != 0:
- print(p.stdout)
-
- return ret
+# Looking for the list of excluded tests? See linters.py !
def main() -> None:
@@ -133,7 +31,7 @@ def main() -> None:
if shutil.which(linter) is None:
iotests.notrun(f'{linter} not found')
- files = get_test_files()
+ files = linters.get_test_files()
iotests.logger.debug('Files to be checked:')
iotests.logger.debug(', '.join(sorted(files)))
@@ -148,7 +46,7 @@ def main() -> None:
env['MYPYPATH'] = env['PYTHONPATH']
- run_linters(files, env=env)
+ linters.run_linters(files, env=env)
iotests.script_main(main)
diff --git a/tests/qemu-iotests/linters.py b/tests/qemu-iotests/linters.py
new file mode 100755
index 00000000000..e7e8baded02
--- /dev/null
+++ b/tests/qemu-iotests/linters.py
@@ -0,0 +1,121 @@
+# Copyright (C) 2020 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import subprocess
+import sys
+from typing import List, Mapping, Optional
+
+
+# TODO: Empty this list!
+SKIP_FILES = (
+ '030', '040', '041', '044', '045', '055', '056', '057', '065', '093',
+ '096', '118', '124', '132', '136', '139', '147', '148', '149',
+ '151', '152', '155', '163', '165', '169', '194', '196', '199', '202',
+ '203', '205', '206', '207', '208', '210', '211', '212', '213', '216',
+ '218', '219', '222', '224', '228', '234', '235', '236', '237', '238',
+ '240', '242', '245', '246', '248', '255', '256', '257', '258', '260',
+ '262', '264', '266', '274', '277', '280', '281', '295', '296', '298',
+ '299', '302', '303', '304', '307',
+ 'nbd-fault-injector.py', 'qcow2.py', 'qcow2_format.py', 'qed.py'
+)
+
+
+def is_python_file(filepath: str) -> bool:
+ if filepath.endswith('.py'):
+ return True
+
+ with open(filepath) as f:
+ try:
+ first_line = f.readline()
+ return re.match('^#!.*python', first_line) is not None
+ except UnicodeDecodeError: # Ignore binary files
+ return False
+
+
+def get_test_files(directory: str = '.') -> List[str]:
+ files = []
+
+ for dirent in os.scandir(directory):
+ if dirent.name in SKIP_FILES:
+ continue
+
+ relpath = os.path.join(directory, dirent.name)
+ if dirent.is_dir():
+ files.extend(get_test_files(relpath))
+ elif is_python_file(relpath):
+ files.append(relpath)
+
+ return files
+
+
+def run_linters(
+ files: List[str],
+ directory: str = '.',
+ env: Optional[Mapping[str, str]] = None,
+) -> int:
+ ret = 0
+
+ print('=== pylint ===')
+ sys.stdout.flush()
+
+ # Todo notes are fine, but fixme's or xxx's should probably just be
+ # fixed (in tests, at least)
+ p = subprocess.run(
+ ('python3', '-m', 'pylint', '--score=n', '--notes=FIXME,XXX', *files),
+ cwd=directory,
+ env=env,
+ check=False,
+ universal_newlines=True,
+ )
+ ret += p.returncode
+
+ print('=== mypy ===')
+ sys.stdout.flush()
+
+ # We have to call mypy separately for each file. Otherwise, it
+ # will interpret all given files as belonging together (i.e., they
+ # may not both define the same classes, etc.; most notably, they
+ # must not both define the __main__ module).
+ for filename in files:
+ p = subprocess.run(
+ (
+ 'python3', '-m', 'mypy',
+ '--warn-unused-configs',
+ '--disallow-subclassing-any',
+ '--disallow-any-generics',
+ '--disallow-incomplete-defs',
+ '--disallow-untyped-decorators',
+ '--no-implicit-optional',
+ '--warn-redundant-casts',
+ '--warn-unused-ignores',
+ '--no-implicit-reexport',
+ '--namespace-packages',
+ filename,
+ ),
+ cwd=directory,
+ env=env,
+ check=False,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True
+ )
+
+ ret += p.returncode
+ if p.returncode != 0:
+ print(p.stdout)
+
+ return ret
--
2.31.1
- [PATCH v2 06/17] iotests/297: modify is_python_file to work from any CWD, (continued)
- [PATCH v2 06/17] iotests/297: modify is_python_file to work from any CWD, John Snow, 2021/07/20
- [PATCH v2 05/17] iotests/migrate-bitmaps-test: delint, John Snow, 2021/07/20
- [PATCH v2 07/17] iotests/297: Add get_files() function, John Snow, 2021/07/20
- [PATCH v2 08/17] iotests/297: Include sub-directories when finding tests to lint, John Snow, 2021/07/20
- [PATCH v2 09/17] iotests/297: Don't rely on distro-specific linter binaries, John Snow, 2021/07/20
- [PATCH v2 11/17] iotests/297: Separate environment setup from test execution, John Snow, 2021/07/20
- [PATCH v2 10/17] iotests/297: Create main() function, John Snow, 2021/07/20
- [PATCH v2 13/17] iotests/297: return error code from run_linters(), John Snow, 2021/07/20
- [PATCH v2 12/17] iotests/297: Add 'directory' argument to run_linters, John Snow, 2021/07/20
- [PATCH v2 14/17] iotests/297: split linters.py off from 297,
John Snow <=
- [PATCH v2 15/17] iotests/linters: Add entry point for Python CI linters, John Snow, 2021/07/20
- [PATCH v2 16/17] python: Add iotest linters to test suite, John Snow, 2021/07/20
- [PATCH v2 17/17] iotests/linters: check mypy files all at once, John Snow, 2021/07/20
- Re: [PATCH v2 00/17] python/iotests: Run iotest linters during Python CI, John Snow, 2021/07/20