From fc61f0faa28ec187c155d3a48ae36d3fdca9d97a Mon Sep 17 00:00:00 2001 From: appel_c <christian.appel@psi.ch> Date: Sat, 20 Apr 2024 22:23:28 +0200 Subject: [PATCH 1/5] feat: add package tests to for night runs to test min, latest and random dep. packages --- noxfile.py | 66 ++++++++ ophyd_devices/package_version_handler.py | 187 +++++++++++++++++++++++ pyproject.toml | 3 +- 3 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 noxfile.py create mode 100644 ophyd_devices/package_version_handler.py diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..eaedb101 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,66 @@ +""" This script is used by the CI to check dependencies, +and run tests with minimum required package versions.""" + +import os +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path + +import nox +from pip import main + +from ophyd_devices.package_version_handler import Operation, PackageVersionHandler + +# Get package version from pyproject.toml +basepath = str(Path(__file__).resolve().parent) +toml_filepath = os.path.join(basepath, "pyproject.toml") +with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=True) as handler: + toml_min = handler.get_toml(Operation.MIN) + toml_latest = handler.get_toml(Operation.MAX) + toml_random = handler.get_toml(Operation.RANDOM) + + handler.write_toml_file("./pyproject.toml", toml_min) + + @nox.session + def test_minimum_versions(session): + """Test the project with the minimal required versions of the dependencies.""" + session.install(".[dev]") + session.run("pytest", "-v", "--random-order", "tests/") + + handler.write_toml_file("./pyproject.toml", toml_latest) + + @nox.session + def test_latest_versions(session): + """Test the project with the minimal required versions of the dependencies.""" + session.install(".[dev]") + session.run("pytest", "-v", "--random-order", "tests/") + + handler.write_toml_file("./pyproject.toml", toml_random) + + @nox.session + def test_random_versions(session): + """Test the project with the minimal required versions of the dependencies.""" + session.install(".[dev]") + session.run("pytest", "-v", "--random-order", "tests/") + + +# # Finally, parse the output from pip list --outdate +# # only including the packages in the pyproject.toml file +# parser = StringIO() + +# with redirect_stdout(parser): +# main(["list", "--outdated"]) + +# installed_dep = {} +# most_recent_dep = {} +# for line in parser.getvalue().split("\n"): +# if line.startswith("Package") or line.startswith("-----"): +# continue +# if line: +# line = line.split() +# if line[0] in handler.available_candidates: +# installed_dep[line[0]] = tuple(line[1].split(".")) +# most_recent_dep[line[0]] = tuple(line[2].split(".")) + +# for key, value in installed_dep.items(): +# print(f"{key}: {value} -> {most_recent_dep[key]} : {handler.available_candidates[key]}") diff --git a/ophyd_devices/package_version_handler.py b/ophyd_devices/package_version_handler.py new file mode 100644 index 00000000..4ad96bff --- /dev/null +++ b/ophyd_devices/package_version_handler.py @@ -0,0 +1,187 @@ +""" Module for handling package versions from pyproject.toml file. """ + +import enum +import os +import random +import warnings +from copy import deepcopy +from typing import DefaultDict, Literal + +import tomlkit.api +from packaging.requirements import Requirement +from packaging.version import Version +from piptools.repositories.pypi import PyPIRepository +from tomlkit.toml_document import TOMLDocument + + +class Operation(str, enum.Enum): + """Operator class.""" + + MIN = "min" + MAX = "max" + RANDOM = "random" + + +class PackageVersionHandler: + """PackageVersionHandler class.""" + + def __init__(self, toml_filepath: str, include_optional_dep: bool = True): + """Initialize the PackageVersionHandler class. + + Ths class should be used as a context manager + + This class is used to load available package versions from PyPI + based on the dependencies in the pyproject.toml file. + It can provide the minimum required versions for each package, + the maximum required versions for each package or a random + distribution of versions for each package used for CI pipelines. + + Args: + toml_filepath (str): Path to the pyproject.toml file. + include_optional_dep (bool): Include optional dependencies. Defaults to True. + + Example: + >>> with PackageVersionHandler(toml_file="./pyproject.toml") as handler: + >>> toml_min = handler.get_toml_min_req() + >>> toml_latest = handler.get_toml_latest_req() + >>> toml_random = handler.get_toml_random_req() + >>> handler.write_toml_file("requirements_min.txt", min_req) + + """ + self.repo = PyPIRepository(pip_args=[""], cache_dir="") + self._toml_filepath = toml_filepath + self._project_toml_save_copy = None + self._available_candidates = {} + self._pkg_dep = {} + self._include_optional_dep = include_optional_dep + + def __enter__(self): + self._project_toml_save_copy = self.load_toml_file(self._toml_filepath) + self.run() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.write_toml_file(self._toml_filepath, self._project_toml_save_copy) + + def get_project_toml(self): + """Create copy from the save_copy of the project toml file.""" + return deepcopy(self._project_toml_save_copy) + + @property + def available_candidates(self) -> dict: + """Returns dictionary with available candidates for each package.""" + return self._available_candidates + + def get_toml(self, operation: Operation) -> TOMLDocument: + """Returns the minimum required versions for each package. + + Args: + operation (Operation): Option to pick "min", "max" or "random" versions. + """ + toml = self.get_project_toml() + dependencies = self._select_versions(operation=operation) + for name, dep_list in dependencies.items(): + if name == "core_dep": + toml["project"]["dependencies"] = dependencies["core_dep"] + continue + toml["project"]["optional-dependencies"][name] = dependencies[name] + return toml + + def _select_versions(self, operation: Operation) -> dict: + """Select versions based on the input argument. + + Args: + operation (Literal["min", "max", "random"]): Operation to perform on the available candidates. + """ + if operation == Operation.MIN: + op = min + elif operation == Operation.MAX: + op = max + elif operation == Operation.RANDOM: + op = random.choice + else: + raise ValueError( + f"Invalid operation: {operation}. Choose from 'min', 'max' or 'random'." + ) + dependencies = DefaultDict(lambda: []) + for name, dep in self._available_candidates.items(): + for dep_name, dep_version in dep.items(): + version = op(dep_version) + dependencies[name].extend([f"{dep_name}=={version.public}"]) + return dependencies + + def load_toml_file(self, filepath: str) -> TOMLDocument: + """load toml file and return the content as a dictionary.""" + filepath = os.path.abspath(filepath) + with open(filepath) as f: + return tomlkit.api.load(f) + + def write_toml_file(self, filepath: str, toml_dict: TOMLDocument) -> None: + """Save toml file to disk.""" + filepath = os.path.abspath(filepath) + with open(filepath, "w") as f: + tomlkit.api.dump(toml_dict, f) + + def get_package_dependencies_from_toml(self, include_optional_dep: bool = True) -> dict: + """Gets the package dependencies from the pyproject.toml file. + + Args: + filepath (str): Path to the pyproject.toml file. + include_optional_dep (bool): Include optional dependencies. Defaults to True. + """ + project_toml = self.get_project_toml() + self._pkg_dep["core_dep"] = project_toml["project"]["dependencies"] + if include_optional_dep: + if "optional-dependencies" not in project_toml["project"]: + return self._pkg_dep + for name, dep in project_toml["project"]["optional-dependencies"].items(): + self._pkg_dep[name] = dep + return self._pkg_dep + + def get_available_candidates(self, pkg_dep: dict) -> dict: + """Update available candidates from dependency list. + + Args: + dependency_list (list[str]): List of dependencies as parsed from pyproject.toml file. + """ + rtr = {} + for name, dep in pkg_dep.items(): + rtr[name] = {} + for req_str in dep: + rtr[name].update(self.update_candidate(req_str)) + return rtr + + def update_candidate(self, req_str: str) -> dict: + """Update candidate information in self._available_candidates. + + Args: + req_str (str): Requirement string in the format from toml. + """ + ireq = Requirement(req_str) + all_candidates = set( + [ + Version(candidate.version.public) + for candidate in self.repo.find_all_candidates(ireq.name) + ] + ) + return {ireq.name: sorted(list(ireq.specifier.filter(all_candidates, prereleases=False)))} + + def run(self) -> None: + """Run the package version handler.""" + self._pkg_dep = self.get_package_dependencies_from_toml( + include_optional_dep=self._include_optional_dep + ) + self._available_candidates = self.get_available_candidates(self._pkg_dep) + + +if __name__ == "__main__": + + with PackageVersionHandler(toml_filepath="./pyproject.toml") as handler: + toml_min = handler.get_toml(Operation.MIN) + toml_latest = handler.get_toml(Operation.MAX) + toml_random = handler.get_toml(Operation.RANDOM) + handler.write_toml_file("./pyproject.toml", toml_min) + handler.write_toml_file("./requirements_latest.toml", toml_latest) + handler.write_toml_file("./requirements_random.toml", toml_random) + + print("Done!") diff --git a/pyproject.toml b/pyproject.toml index 3d3af6e3..0300a7d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,9 +13,10 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ - "ophyd ~= 1.9, <= 1.9.5", + "ophyd ~= 1.9", "typeguard ~= 4.0", "prettytable ~= 3.9", + "bec_server", "bec_lib ~= 2.0", "numpy ~= 1.24", "pyyaml ~= 6.0", -- GitLab From 1a63e13dcc99bbc3a3befa361513f6ff1bdd6040 Mon Sep 17 00:00:00 2001 From: appel_c <christian.appel@psi.ch> Date: Sat, 20 Apr 2024 22:42:54 +0200 Subject: [PATCH 2/5] refactor: add job to ci pipeline --- .gitlab-ci.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30c45f97..ad52125a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,6 @@ workflow: include: - template: Security/Secret-Detection.gitlab-ci.yml - #commands to run in the Docker container before starting each job. before_script: - pip install -e .[dev] @@ -37,7 +36,7 @@ stages: formatter: stage: Formatter before_script: - - '' + - "" script: - pip install black isort - pip install -e .[dev] @@ -69,10 +68,10 @@ pylint-check: script: # Identify changed Python files - if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then - TARGET_BRANCH_COMMIT_SHA=$(git rev-parse $CI_MERGE_REQUEST_TARGET_BRANCH_NAME); - CHANGED_FILES=$(git diff --name-only $SOURCE_BRANCH_COMMIT_SHA $TARGET_BRANCH_COMMIT_SHA | grep '\.py$' || true); + TARGET_BRANCH_COMMIT_SHA=$(git rev-parse $CI_MERGE_REQUEST_TARGET_BRANCH_NAME); + CHANGED_FILES=$(git diff --name-only $SOURCE_BRANCH_COMMIT_SHA $TARGET_BRANCH_COMMIT_SHA | grep '\.py$' || true); else - CHANGED_FILES=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA | grep '\.py$' || true); + CHANGED_FILES=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA | grep '\.py$' || true); fi - if [ -z "$CHANGED_FILES" ]; then echo "No Python files changed."; exit 0; fi @@ -91,7 +90,7 @@ pylint-check: secret_detection: before_script: - - '' + - "" pytest: stage: test @@ -109,7 +108,7 @@ pytest: path: coverage.xml config_test: - stage: test + stage: test script: - ophyd_test --config ./ophyd_devices/epics/db/ --output ./config_tests artifacts: @@ -142,6 +141,14 @@ trigger_bec: rules: - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"' +night_tests: + stage: AdditionalTests + script: + - pip install nox + - nox + rules: + - if: '$TEST_PACKAGES == "1"' + semver: stage: Deploy needs: ["pytest"] -- GitLab From 208309821b61ea97d865e9c93838a2f8cf2b78e5 Mon Sep 17 00:00:00 2001 From: appel_c <christian.appel@psi.ch> Date: Mon, 22 Apr 2024 13:06:56 +0200 Subject: [PATCH 3/5] wip --- .gitlab-ci.yml | 1 + noxfile.py | 40 +++++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad52125a..f94fc239 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -144,6 +144,7 @@ trigger_bec: night_tests: stage: AdditionalTests script: + - pip install pip-tools - pip install nox - nox rules: diff --git a/noxfile.py b/noxfile.py index eaedb101..039af4b0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,9 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru toml_latest = handler.get_toml(Operation.MAX) toml_random = handler.get_toml(Operation.RANDOM) + print("Starting tests with ") handler.write_toml_file("./pyproject.toml", toml_min) + print(f"Installing dependencies from minimum requirements: \n{toml_min}") @nox.session def test_minimum_versions(session): @@ -28,6 +30,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru session.run("pytest", "-v", "--random-order", "tests/") handler.write_toml_file("./pyproject.toml", toml_latest) + print(f"Installing dependencies from latest requirements: \n{toml_latest}") @nox.session def test_latest_versions(session): @@ -36,6 +39,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru session.run("pytest", "-v", "--random-order", "tests/") handler.write_toml_file("./pyproject.toml", toml_random) + print(f"Installing dependencies from random distribution of requirements: \n{toml_random}") @nox.session def test_random_versions(session): @@ -44,23 +48,25 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru session.run("pytest", "-v", "--random-order", "tests/") -# # Finally, parse the output from pip list --outdate -# # only including the packages in the pyproject.toml file -# parser = StringIO() +# Finally, parse the output from pip list --outdate +# only including the packages in the pyproject.toml file +parser = StringIO() -# with redirect_stdout(parser): -# main(["list", "--outdated"]) +with redirect_stdout(parser): + main(["list", "--outdated"]) -# installed_dep = {} -# most_recent_dep = {} -# for line in parser.getvalue().split("\n"): -# if line.startswith("Package") or line.startswith("-----"): -# continue -# if line: -# line = line.split() -# if line[0] in handler.available_candidates: -# installed_dep[line[0]] = tuple(line[1].split(".")) -# most_recent_dep[line[0]] = tuple(line[2].split(".")) +installed_dep = {} +most_recent_dep = {} +for line in parser.getvalue().split("\n"): + if line.startswith("Package") or line.startswith("-----"): + continue + if line: + line = line.split() + if line[0] in handler.available_candidates: + installed_dep[line[0]] = tuple(line[1].split(".")) + most_recent_dep[line[0]] = tuple(line[2].split(".")) -# for key, value in installed_dep.items(): -# print(f"{key}: {value} -> {most_recent_dep[key]} : {handler.available_candidates[key]}") +print("Summary of outdated package dependencies:") +print("-----------------------------------------") +for key, value in installed_dep.items(): + print(f"{key}: {value} -> {most_recent_dep[key]} : {handler.available_candidates[key]}") -- GitLab From d5e17e2c897173efa06d0abd55845094142cc1df Mon Sep 17 00:00:00 2001 From: appel_c <christian.appel@psi.ch> Date: Mon, 22 Apr 2024 13:22:04 +0200 Subject: [PATCH 4/5] wip --- noxfile.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 039af4b0..3a8eb154 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,14 +19,15 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru toml_latest = handler.get_toml(Operation.MAX) toml_random = handler.get_toml(Operation.RANDOM) - print("Starting tests with ") handler.write_toml_file("./pyproject.toml", toml_min) - print(f"Installing dependencies from minimum requirements: \n{toml_min}") @nox.session def test_minimum_versions(session): """Test the project with the minimal required versions of the dependencies.""" + + print(f"Installing dependencies from minimum requirements: \n{toml_min}") session.install(".[dev]") + session.run("pip", "list") session.run("pytest", "-v", "--random-order", "tests/") handler.write_toml_file("./pyproject.toml", toml_latest) @@ -36,6 +37,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru def test_latest_versions(session): """Test the project with the minimal required versions of the dependencies.""" session.install(".[dev]") + session.run("pip", "list") session.run("pytest", "-v", "--random-order", "tests/") handler.write_toml_file("./pyproject.toml", toml_random) @@ -45,6 +47,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru def test_random_versions(session): """Test the project with the minimal required versions of the dependencies.""" session.install(".[dev]") + session.run("pip", "list") session.run("pytest", "-v", "--random-order", "tests/") -- GitLab From da5e808ed732238544e1c6dc60294df21cab5a2e Mon Sep 17 00:00:00 2001 From: appel_c <christian.appel@psi.ch> Date: Mon, 22 Apr 2024 13:22:30 +0200 Subject: [PATCH 5/5] wip --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f94fc239..74c037cd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,13 +142,13 @@ trigger_bec: - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"' night_tests: - stage: AdditionalTests + stage: test script: - pip install pip-tools - pip install nox - nox - rules: - - if: '$TEST_PACKAGES == "1"' + # rules: + # - if: '$TEST_PACKAGES == "1"' semver: stage: Deploy -- GitLab