Feature/holes reproduction

.github/workflows/publish.yml

@@ -1,13 +1,11 @@
-name: Publish Package on PYPI
+name: Publish Package on PyPI
 
 on:
   release:
     types: [published]
 
-
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
 
     steps:
@@ -16,14 +14,19 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        poetry install
     - name: Build package
-      run: python setup.py sdist bdist_wheel
+      run: |
+        poetry build
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
+      env:
+        PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        poetry config pypi-token.pypi $PYPI_TOKEN
+        poetry publish

.github/workflows/test.yml

@@ -1,45 +1,102 @@
-name: Unit tests
+name: Unit tests and docs generation
 
 on:
   push:
     branches:
-      -dev
-      -main
+      - "**"
   pull_request:
+    branches:
+      - "**"
     types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
 
 jobs:
-  build-linux:
+  check:
     if: github.event.pull_request.draft == false
-    runs-on: ${{matrix.os}}
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
-      - name: Git clone
+      - name: Set OS and Python version
+        id: set-vars
+        run: |
+          if [[ "${GITHUB_REF}" == "refs/heads/main" || "${GITHUB_REF}" == "refs/heads/dev" ]]; then
+            echo "os-matrix=ubuntu-latest,windows-latest" >> $GITHUB_ENV
+            echo "python-matrix=3.8,3.9,3.10,3.11" >> $GITHUB_ENV
+          else
+            echo "os-matrix=ubuntu-latest" >> $GITHUB_ENV
+            echo "python-matrix=3.11" >> $GITHUB_ENV
+          fi
+      - name: Checkout
         uses: actions/checkout@v3
-      - name: Set up venv for ci
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Python
+        uses: actions/setup-python@v4
         with:
-          python-version: ${{matrix.python-version}}
-          environment-file: environment.ci.yml
-      - name: Lint with flake8
-        run: |
-          flake8
-      - name: Test with pytest
-        run: |
-          make coverage
-      - name: typing with mypy
-        run: |
-          mypy qolmat
-          echo you should uncomment mypy qolmat and delete this line
-      - name: Upload coverage reports to Codecov
+          python-version: ${{ matrix.python-version }}
+      - name: Cache Poetry
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-${{ matrix.python-version }}-
+      - name: Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.3
+      - name: Lock
+        run: poetry lock --no-update
+      - name: Install
+        run: poetry install
+      - name: Checkers
+        run: make checkers
+      - name: Codecov
         uses: codecov/codecov-action@v3
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  docs:
+    runs-on: ubuntu-latest
+    needs: check
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache Poetry
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-${{ matrix.python-version }}-
+      - name: Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.3
+      - name: Lock
+        run: poetry lock --no-update
+      - name: Install
+        run: poetry install
+      - name: Check Changed Files
+        id: changed-files
+        run: |
+          git fetch origin ${{ github.base_ref }}:${{ github.base_ref }} --depth=1
+          git diff --name-only ${{ github.base_ref }} > changed_files.txt
+      - name: Build Docs
+        if: contains(fromJSON('["docs/", ".rst"]').join(','), fromJSON('["${{ steps.changed-files.outputs.files }}"]').join(','))
+        run: |
+          poetry run sphinx-build -b html docs/ _build/html

.gitignore

@@ -28,6 +28,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+poetry.lock
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

.pre-commit-config.yaml

@@ -8,20 +8,8 @@ repos:
         exclude: (docs/)
       - id: trailing-whitespace
         exclude: (docs/)
-  - repo: https://github.com/psf/black
-    rev: 22.8.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.3
     hooks:
-      - id: black
-        args:
-          - "-l 99"
-  # Flake8
-  - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-      - id: flake8
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
-    hooks:
-      - id: mypy
-        args: [--ignore-missing-imports]
-        additional_dependencies: [types-requests]
+      - id: ruff
+      - id: ruff-format

CONTRIBUTING.rst

@@ -29,14 +29,11 @@ You can create a virtual environment via `conda`:
 
 .. code:: sh
 
-    $ conda env create -f environment.dev.yml
-    $ conda activate env_qolmat_dev
-
-If you need to use pytorch, enter the command:
-
-.. code:: sh
-
-    $ pip install -e .[pytorch]
+    $ pip install poetry
+    $ poetry config virtualenvs.in-project true
+    $ poetry lock
+    $ poetry install
+    $ poetry shell
 
 Once the environment is installed, pre-commit is installed, but need to be activated using the following command:
 
@@ -78,7 +75,7 @@ These tests absolutely have to pass.
 
 .. code:: sh
 
-    $ mypy qolmat
+    $ make check-types
 
 Unit test
 ^^^^^^^^^
@@ -88,4 +85,4 @@ The coverage should on new features must be above 95%.
 
 .. code:: sh
 
-    $ pytest -vs --cov-branch --cov=qolmat --pyargs tests --cov-report term-missing
+    $ make check-coverage

Makefile

@@ -1,13 +1,34 @@
-coverage:
-	pytest --cov-branch --cov=qolmat --cov-report=xml tests
 
-doctest:
-	pytest --doctest-modules --pyargs qolmat
+check-coverage:
+	poetry run pytest --cov-branch --cov=qolmat/ --cov-report=xml tests/
 
-doc:
-	make html -C docs
+check-poetry:
+	poetry check --lock
+
+check-quality:
+	poetry run ruff check qolmat/ tests/
+
+check-security:
+	poetry run bandit --recursive --configfile=pyproject.toml qolmat/
+
+check-tests:
+	poetry run pytest tests/
+
+check-types:
+	poetry run mypy qolmat/ tests/
+
+checkers: check-coverage check-types
 
 clean:
 	rm -rf .mypy_cache .pytest_cache .coverage*
 	rm -rf **__pycache__
 	make clean -C docs
+
+coverage:
+	poetry run pytest --cov-branch --cov=qolmat --cov-report=xml tests
+
+doc:
+	make html -C docs
+
+doctest:
+	poetry run pytest --doctest-modules --pyargs qolmat

README.rst

@@ -70,14 +70,13 @@ With just these few lines of code, you can see how easy it is to
   from qolmat.utils import data
 
   # load and prepare csv data
-
   df_data = data.get_data("Beijing")
   columns = ["TEMP", "PRES", "WSPM"]
   df_data = df_data[columns]
   df_with_nan = data.add_holes(df_data, ratio_masked=0.2, mean_size=120)
 
   # impute and compare
-  imputer_mean = imputers.ImputerMean(groups=("station",))
+  imputer_mean = imputers.ImputerSimple(strategy="mean", groups=("station",))
   imputer_interpol = imputers.ImputerInterpolation(method="linear", groups=("station",))
   imputer_var1 = imputers.ImputerEM(model="VAR", groups=("station",), method="mle", max_iter_em=50, n_iter_ou=15, dt=1e-3, p=1)
   dict_imputers = {
@@ -90,7 +89,7 @@ With just these few lines of code, you can see how easy it is to
         dict_imputers,
         columns,
         generator_holes = generator_holes,
-        metrics = ["mae", "wmape", "KL_columnwise", "ks_test", "energy"],
+        metrics = ["mae", "wmape", "kl_columnwise", "ks_test", "energy"],
     )
   results = comparison.compare(df_with_nan)
   results.style.highlight_min(color="lightsteelblue", axis=1)

docs/Makefile

@@ -37,6 +37,9 @@ clean:
 	-rm -rf examples/tutorials/*
 	-rm -rf generated/*
 
+doctest:
+	$(SPHINXBUILD) -b doctest . _build/doctest
+
 html:
 	# These two lines make the build a bit more lengthy, and the
 	# the embedding of images more robust