From fc61f0faa28ec187c155d3a48ae36d3fdca9d97a Mon Sep 17 00:00:00 2001
From: appel_c <christian.appel@psi.ch>
Date: Sat, 20 Apr 2024 22:23:28 +0200
Subject: [PATCH 1/5] feat: add package tests to for night runs to test min,
 latest and random dep. packages

---
 noxfile.py                               |  66 ++++++++
 ophyd_devices/package_version_handler.py | 187 +++++++++++++++++++++++
 pyproject.toml                           |   3 +-
 3 files changed, 255 insertions(+), 1 deletion(-)
 create mode 100644 noxfile.py
 create mode 100644 ophyd_devices/package_version_handler.py

diff --git a/noxfile.py b/noxfile.py
new file mode 100644
index 00000000..eaedb101
--- /dev/null
+++ b/noxfile.py
@@ -0,0 +1,66 @@
+""" This script is used by the CI to check dependencies, 
+and run tests with minimum required package versions."""
+
+import os
+from contextlib import redirect_stdout
+from io import StringIO
+from pathlib import Path
+
+import nox
+from pip import main
+
+from ophyd_devices.package_version_handler import Operation, PackageVersionHandler
+
+# Get package version from pyproject.toml
+basepath = str(Path(__file__).resolve().parent)
+toml_filepath = os.path.join(basepath, "pyproject.toml")
+with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=True) as handler:
+    toml_min = handler.get_toml(Operation.MIN)
+    toml_latest = handler.get_toml(Operation.MAX)
+    toml_random = handler.get_toml(Operation.RANDOM)
+
+    handler.write_toml_file("./pyproject.toml", toml_min)
+
+    @nox.session
+    def test_minimum_versions(session):
+        """Test the project with the minimal required versions of the dependencies."""
+        session.install(".[dev]")
+        session.run("pytest", "-v", "--random-order", "tests/")
+
+    handler.write_toml_file("./pyproject.toml", toml_latest)
+
+    @nox.session
+    def test_latest_versions(session):
+        """Test the project with the minimal required versions of the dependencies."""
+        session.install(".[dev]")
+        session.run("pytest", "-v", "--random-order", "tests/")
+
+    handler.write_toml_file("./pyproject.toml", toml_random)
+
+    @nox.session
+    def test_random_versions(session):
+        """Test the project with the minimal required versions of the dependencies."""
+        session.install(".[dev]")
+        session.run("pytest", "-v", "--random-order", "tests/")
+
+
+# # Finally, parse the output from pip list --outdate
+# # only including the packages in the pyproject.toml file
+# parser = StringIO()
+
+# with redirect_stdout(parser):
+#     main(["list", "--outdated"])
+
+# installed_dep = {}
+# most_recent_dep = {}
+# for line in parser.getvalue().split("\n"):
+#     if line.startswith("Package") or line.startswith("-----"):
+#         continue
+#     if line:
+#         line = line.split()
+#         if line[0] in handler.available_candidates:
+#             installed_dep[line[0]] = tuple(line[1].split("."))
+#             most_recent_dep[line[0]] = tuple(line[2].split("."))
+
+# for key, value in installed_dep.items():
+#     print(f"{key}: {value} -> {most_recent_dep[key]} : {handler.available_candidates[key]}")
diff --git a/ophyd_devices/package_version_handler.py b/ophyd_devices/package_version_handler.py
new file mode 100644
index 00000000..4ad96bff
--- /dev/null
+++ b/ophyd_devices/package_version_handler.py
@@ -0,0 +1,187 @@
+""" Module for handling package versions from pyproject.toml file. """
+
+import enum
+import os
+import random
+import warnings
+from copy import deepcopy
+from typing import DefaultDict, Literal
+
+import tomlkit.api
+from packaging.requirements import Requirement
+from packaging.version import Version
+from piptools.repositories.pypi import PyPIRepository
+from tomlkit.toml_document import TOMLDocument
+
+
+class Operation(str, enum.Enum):
+    """Operator class."""
+
+    MIN = "min"
+    MAX = "max"
+    RANDOM = "random"
+
+
+class PackageVersionHandler:
+    """PackageVersionHandler class."""
+
+    def __init__(self, toml_filepath: str, include_optional_dep: bool = True):
+        """Initialize the PackageVersionHandler class.
+
+        Ths class should be used as a context manager
+
+        This class is used to load available package versions from PyPI
+        based on the dependencies in the pyproject.toml file.
+        It can provide the minimum required versions for each package,
+        the maximum required versions for each package or a random
+        distribution of versions for each package used for CI pipelines.
+
+        Args:
+            toml_filepath (str): Path to the pyproject.toml file.
+            include_optional_dep (bool): Include optional dependencies. Defaults to True.
+
+        Example:
+        >>> with PackageVersionHandler(toml_file="./pyproject.toml") as handler:
+        >>>     toml_min = handler.get_toml_min_req()
+        >>>     toml_latest = handler.get_toml_latest_req()
+        >>>     toml_random = handler.get_toml_random_req()
+        >>>     handler.write_toml_file("requirements_min.txt", min_req)
+
+        """
+        self.repo = PyPIRepository(pip_args=[""], cache_dir="")
+        self._toml_filepath = toml_filepath
+        self._project_toml_save_copy = None
+        self._available_candidates = {}
+        self._pkg_dep = {}
+        self._include_optional_dep = include_optional_dep
+
+    def __enter__(self):
+        self._project_toml_save_copy = self.load_toml_file(self._toml_filepath)
+        self.run()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.write_toml_file(self._toml_filepath, self._project_toml_save_copy)
+
+    def get_project_toml(self):
+        """Create copy from the save_copy of the project toml file."""
+        return deepcopy(self._project_toml_save_copy)
+
+    @property
+    def available_candidates(self) -> dict:
+        """Returns dictionary with available candidates for each package."""
+        return self._available_candidates
+
+    def get_toml(self, operation: Operation) -> TOMLDocument:
+        """Returns the minimum required versions for each package.
+
+        Args:
+            operation (Operation): Option to pick "min", "max" or "random" versions.
+        """
+        toml = self.get_project_toml()
+        dependencies = self._select_versions(operation=operation)
+        for name, dep_list in dependencies.items():
+            if name == "core_dep":
+                toml["project"]["dependencies"] = dependencies["core_dep"]
+                continue
+            toml["project"]["optional-dependencies"][name] = dependencies[name]
+        return toml
+
+    def _select_versions(self, operation: Operation) -> dict:
+        """Select versions based on the input argument.
+
+        Args:
+            operation (Literal["min", "max", "random"]): Operation to perform on the available candidates.
+        """
+        if operation == Operation.MIN:
+            op = min
+        elif operation == Operation.MAX:
+            op = max
+        elif operation == Operation.RANDOM:
+            op = random.choice
+        else:
+            raise ValueError(
+                f"Invalid operation: {operation}. Choose from 'min', 'max' or 'random'."
+            )
+        dependencies = DefaultDict(lambda: [])
+        for name, dep in self._available_candidates.items():
+            for dep_name, dep_version in dep.items():
+                version = op(dep_version)
+                dependencies[name].extend([f"{dep_name}=={version.public}"])
+        return dependencies
+
+    def load_toml_file(self, filepath: str) -> TOMLDocument:
+        """load toml file and return the content as a dictionary."""
+        filepath = os.path.abspath(filepath)
+        with open(filepath) as f:
+            return tomlkit.api.load(f)
+
+    def write_toml_file(self, filepath: str, toml_dict: TOMLDocument) -> None:
+        """Save toml file to disk."""
+        filepath = os.path.abspath(filepath)
+        with open(filepath, "w") as f:
+            tomlkit.api.dump(toml_dict, f)
+
+    def get_package_dependencies_from_toml(self, include_optional_dep: bool = True) -> dict:
+        """Gets the package dependencies from the pyproject.toml file.
+
+        Args:
+            filepath (str): Path to the pyproject.toml file.
+            include_optional_dep (bool): Include optional dependencies. Defaults to True.
+        """
+        project_toml = self.get_project_toml()
+        self._pkg_dep["core_dep"] = project_toml["project"]["dependencies"]
+        if include_optional_dep:
+            if "optional-dependencies" not in project_toml["project"]:
+                return self._pkg_dep
+            for name, dep in project_toml["project"]["optional-dependencies"].items():
+                self._pkg_dep[name] = dep
+        return self._pkg_dep
+
+    def get_available_candidates(self, pkg_dep: dict) -> dict:
+        """Update available candidates from dependency list.
+
+        Args:
+            dependency_list (list[str]): List of dependencies as parsed from pyproject.toml file.
+        """
+        rtr = {}
+        for name, dep in pkg_dep.items():
+            rtr[name] = {}
+            for req_str in dep:
+                rtr[name].update(self.update_candidate(req_str))
+        return rtr
+
+    def update_candidate(self, req_str: str) -> dict:
+        """Update candidate information in self._available_candidates.
+
+        Args:
+            req_str (str): Requirement string in the format from toml.
+        """
+        ireq = Requirement(req_str)
+        all_candidates = set(
+            [
+                Version(candidate.version.public)
+                for candidate in self.repo.find_all_candidates(ireq.name)
+            ]
+        )
+        return {ireq.name: sorted(list(ireq.specifier.filter(all_candidates, prereleases=False)))}
+
+    def run(self) -> None:
+        """Run the package version handler."""
+        self._pkg_dep = self.get_package_dependencies_from_toml(
+            include_optional_dep=self._include_optional_dep
+        )
+        self._available_candidates = self.get_available_candidates(self._pkg_dep)
+
+
+if __name__ == "__main__":
+
+    with PackageVersionHandler(toml_filepath="./pyproject.toml") as handler:
+        toml_min = handler.get_toml(Operation.MIN)
+        toml_latest = handler.get_toml(Operation.MAX)
+        toml_random = handler.get_toml(Operation.RANDOM)
+        handler.write_toml_file("./pyproject.toml", toml_min)
+        handler.write_toml_file("./requirements_latest.toml", toml_latest)
+        handler.write_toml_file("./requirements_random.toml", toml_random)
+
+    print("Done!")
diff --git a/pyproject.toml b/pyproject.toml
index 3d3af6e3..0300a7d6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,9 +13,10 @@ classifiers = [
     "Topic :: Scientific/Engineering",
 ]
 dependencies = [
-    "ophyd ~= 1.9, <= 1.9.5",
+    "ophyd ~= 1.9",
     "typeguard ~= 4.0",
     "prettytable ~= 3.9",
+    "bec_server",
     "bec_lib ~= 2.0",
     "numpy ~= 1.24",
     "pyyaml ~= 6.0",
-- 
GitLab


From 1a63e13dcc99bbc3a3befa361513f6ff1bdd6040 Mon Sep 17 00:00:00 2001
From: appel_c <christian.appel@psi.ch>
Date: Sat, 20 Apr 2024 22:42:54 +0200
Subject: [PATCH 2/5] refactor: add job to ci pipeline

---
 .gitlab-ci.yml | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 30c45f97..ad52125a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,6 @@ workflow:
 include:
   - template: Security/Secret-Detection.gitlab-ci.yml
 
-
 #commands to run in the Docker container before starting each job.
 before_script:
   - pip install -e .[dev]
@@ -37,7 +36,7 @@ stages:
 formatter:
   stage: Formatter
   before_script:
-    - ''
+    - ""
   script:
     - pip install black isort
     - pip install -e .[dev]
@@ -69,10 +68,10 @@ pylint-check:
   script:
     # Identify changed Python files
     - if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
-        TARGET_BRANCH_COMMIT_SHA=$(git rev-parse $CI_MERGE_REQUEST_TARGET_BRANCH_NAME);
-        CHANGED_FILES=$(git diff --name-only $SOURCE_BRANCH_COMMIT_SHA $TARGET_BRANCH_COMMIT_SHA | grep '\.py$' || true);
+      TARGET_BRANCH_COMMIT_SHA=$(git rev-parse $CI_MERGE_REQUEST_TARGET_BRANCH_NAME);
+      CHANGED_FILES=$(git diff --name-only $SOURCE_BRANCH_COMMIT_SHA $TARGET_BRANCH_COMMIT_SHA | grep '\.py$' || true);
       else
-        CHANGED_FILES=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA | grep '\.py$' || true);
+      CHANGED_FILES=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA | grep '\.py$' || true);
       fi
     - if [ -z "$CHANGED_FILES" ]; then echo "No Python files changed."; exit 0; fi
 
@@ -91,7 +90,7 @@ pylint-check:
 
 secret_detection:
   before_script:
-    - ''
+    - ""
 
 pytest:
   stage: test
@@ -109,7 +108,7 @@ pytest:
         path: coverage.xml
 
 config_test:
-  stage: test 
+  stage: test
   script:
     - ophyd_test --config ./ophyd_devices/epics/db/ --output ./config_tests
   artifacts:
@@ -142,6 +141,14 @@ trigger_bec:
   rules:
     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'
 
+night_tests:
+  stage: AdditionalTests
+  script:
+    - pip install nox
+    - nox
+  rules:
+    - if: '$TEST_PACKAGES == "1"'
+
 semver:
   stage: Deploy
   needs: ["pytest"]
-- 
GitLab


From 208309821b61ea97d865e9c93838a2f8cf2b78e5 Mon Sep 17 00:00:00 2001
From: appel_c <christian.appel@psi.ch>
Date: Mon, 22 Apr 2024 13:06:56 +0200
Subject: [PATCH 3/5] wip

---
 .gitlab-ci.yml |  1 +
 noxfile.py     | 40 +++++++++++++++++++++++-----------------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ad52125a..f94fc239 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -144,6 +144,7 @@ trigger_bec:
 night_tests:
   stage: AdditionalTests
   script:
+    - pip install pip-tools
     - pip install nox
     - nox
   rules:
diff --git a/noxfile.py b/noxfile.py
index eaedb101..039af4b0 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -19,7 +19,9 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
     toml_latest = handler.get_toml(Operation.MAX)
     toml_random = handler.get_toml(Operation.RANDOM)
 
+    print("Starting tests with ")
     handler.write_toml_file("./pyproject.toml", toml_min)
+    print(f"Installing dependencies from minimum requirements: \n{toml_min}")
 
     @nox.session
     def test_minimum_versions(session):
@@ -28,6 +30,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
         session.run("pytest", "-v", "--random-order", "tests/")
 
     handler.write_toml_file("./pyproject.toml", toml_latest)
+    print(f"Installing dependencies from latest requirements: \n{toml_latest}")
 
     @nox.session
     def test_latest_versions(session):
@@ -36,6 +39,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
         session.run("pytest", "-v", "--random-order", "tests/")
 
     handler.write_toml_file("./pyproject.toml", toml_random)
+    print(f"Installing dependencies from random distribution of requirements: \n{toml_random}")
 
     @nox.session
     def test_random_versions(session):
@@ -44,23 +48,25 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
         session.run("pytest", "-v", "--random-order", "tests/")
 
 
-# # Finally, parse the output from pip list --outdate
-# # only including the packages in the pyproject.toml file
-# parser = StringIO()
+# Finally, parse the output from pip list --outdate
+# only including the packages in the pyproject.toml file
+parser = StringIO()
 
-# with redirect_stdout(parser):
-#     main(["list", "--outdated"])
+with redirect_stdout(parser):
+    main(["list", "--outdated"])
 
-# installed_dep = {}
-# most_recent_dep = {}
-# for line in parser.getvalue().split("\n"):
-#     if line.startswith("Package") or line.startswith("-----"):
-#         continue
-#     if line:
-#         line = line.split()
-#         if line[0] in handler.available_candidates:
-#             installed_dep[line[0]] = tuple(line[1].split("."))
-#             most_recent_dep[line[0]] = tuple(line[2].split("."))
+installed_dep = {}
+most_recent_dep = {}
+for line in parser.getvalue().split("\n"):
+    if line.startswith("Package") or line.startswith("-----"):
+        continue
+    if line:
+        line = line.split()
+        if line[0] in handler.available_candidates:
+            installed_dep[line[0]] = tuple(line[1].split("."))
+            most_recent_dep[line[0]] = tuple(line[2].split("."))
 
-# for key, value in installed_dep.items():
-#     print(f"{key}: {value} -> {most_recent_dep[key]} : {handler.available_candidates[key]}")
+print("Summary of outdated package dependencies:")
+print("-----------------------------------------")
+for key, value in installed_dep.items():
+    print(f"{key}: {value} -> {most_recent_dep[key]} : {handler.available_candidates[key]}")
-- 
GitLab


From d5e17e2c897173efa06d0abd55845094142cc1df Mon Sep 17 00:00:00 2001
From: appel_c <christian.appel@psi.ch>
Date: Mon, 22 Apr 2024 13:22:04 +0200
Subject: [PATCH 4/5] wip

---
 noxfile.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/noxfile.py b/noxfile.py
index 039af4b0..3a8eb154 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -19,14 +19,15 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
     toml_latest = handler.get_toml(Operation.MAX)
     toml_random = handler.get_toml(Operation.RANDOM)
 
-    print("Starting tests with ")
     handler.write_toml_file("./pyproject.toml", toml_min)
-    print(f"Installing dependencies from minimum requirements: \n{toml_min}")
 
     @nox.session
     def test_minimum_versions(session):
         """Test the project with the minimal required versions of the dependencies."""
+
+        print(f"Installing dependencies from minimum requirements: \n{toml_min}")
         session.install(".[dev]")
+        session.run("pip", "list")
         session.run("pytest", "-v", "--random-order", "tests/")
 
     handler.write_toml_file("./pyproject.toml", toml_latest)
@@ -36,6 +37,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
     def test_latest_versions(session):
         """Test the project with the minimal required versions of the dependencies."""
         session.install(".[dev]")
+        session.run("pip", "list")
         session.run("pytest", "-v", "--random-order", "tests/")
 
     handler.write_toml_file("./pyproject.toml", toml_random)
@@ -45,6 +47,7 @@ with PackageVersionHandler(toml_filepath=toml_filepath, include_optional_dep=Tru
     def test_random_versions(session):
         """Test the project with the minimal required versions of the dependencies."""
         session.install(".[dev]")
+        session.run("pip", "list")
         session.run("pytest", "-v", "--random-order", "tests/")
 
 
-- 
GitLab


From da5e808ed732238544e1c6dc60294df21cab5a2e Mon Sep 17 00:00:00 2001
From: appel_c <christian.appel@psi.ch>
Date: Mon, 22 Apr 2024 13:22:30 +0200
Subject: [PATCH 5/5] wip

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f94fc239..74c037cd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -142,13 +142,13 @@ trigger_bec:
     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'
 
 night_tests:
-  stage: AdditionalTests
+  stage: test
   script:
     - pip install pip-tools
     - pip install nox
     - nox
-  rules:
-    - if: '$TEST_PACKAGES == "1"'
+  # rules:
+  #   - if: '$TEST_PACKAGES == "1"'
 
 semver:
   stage: Deploy
-- 
GitLab