feat: add auto lint fix tooling with isort and enhanced pre-commit

.github/workflows/test_qlib_from_source.yml

@@ -58,6 +58,10 @@ jobs:
       run: |
         make black
 
+    - name: Check import sorting with isort
+      run: |
+        make isort
+
     - name: Make html with sphinx
       # Since read the docs builds on ubuntu 22.04, we only need to test that the build passes on ubuntu 22.04.
       if: ${{ matrix.os == 'ubuntu-22.04' }}

.pre-commit-config.yaml

@@ -1,12 +1,30 @@
 repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+        exclude: '\.md$'
+    -   id: end-of-file-fixer
+        exclude: '\.md$'
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: check-added-large-files
+        args: ['--maxkb=500']
+
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+
 -   repo: https://github.com/psf/black
     rev: 23.7.0
     hooks:
     -   id: black
-        args: ["qlib", "-l 120"]
+        args: ["-l", "120"]
+        exclude: 'qlib/_version\.py'
 
 -   repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:
-        - id: flake8
-          args: ["--ignore=E501,F541,E266,E402,W503,E731,E203"]
+    -   id: flake8
+        args: ["--ignore=E501,F541,E266,E402,W503,E731,E203", "--per-file-ignores=__init__.py:F401,F403"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean deepclean prerequisite dependencies lightgbm rl develop lint docs package test analysis all install dev black pylint flake8 mypy nbqa nbconvert lint build upload docs-gen
+.PHONY: clean deepclean prerequisite dependencies lightgbm rl develop lint docs package test analysis all install dev black isort pylint flake8 mypy nbqa nbconvert lint fix build upload docs-gen
 #You can modify it according to your terminal
 SHELL := /bin/bash
 
@@ -118,6 +118,10 @@ dev: prerequisite all
 black:
 	black . -l 120 --check --diff --exclude qlib/_version.py
 
+# Check import sorting with isort.
+isort:
+	isort --check-only --diff qlib scripts
+
 # Check code folder with pylint.
 # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
 # 	C0103: invalid-name
@@ -190,7 +194,14 @@ nbqa:
 nbconvert:
 	jupyter nbconvert --to notebook --execute examples/workflow_by_code.ipynb
 
-lint: black pylint flake8 mypy nbqa
+# Run all lint checks.
+lint: black isort pylint flake8 mypy nbqa
+
+# Auto-fix formatting issues (black + isort).
+fix:
+	black . -l 120 --exclude qlib/_version.py
+	isort qlib scripts
+	nbqa black . -l 120
 
 ########################################################################################
 # Package

pyproject.toml

@@ -73,6 +73,7 @@ rl = [
 # which can cause false-positive --check results and inconsistent notebook formatting.
 lint = [
   "black!=26.1.0",
+  "isort",
   "pylint",
   "mypy<1.5.0",
   "flake8",
@@ -124,3 +125,13 @@ qrun = "qlib.cli.run:run"
 local_scheme = "no-local-version"
 version_scheme = "guess-next-dev"
 write_to = "qlib/_version.py"
+
+[tool.isort]
+profile = "black"
+line_length = 120
+known_first_party = ["qlib"]
+skip_glob = ["qlib/_version.py"]
+
+[tool.black]
+line-length = 120
+exclude = "qlib/_version.py"

qlib/backtest/backtest.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from typing import Dict, TYPE_CHECKING, Generator, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Dict, Generator, Optional, Tuple, Union, cast
 
 import pandas as pd
 
@@ -18,7 +18,6 @@
 
 from ..utils.time import Freq
 
-
 PORT_METRIC = Dict[str, Tuple[pd.DataFrame, dict]]
 INDICATOR_METRIC = Dict[str, Tuple[pd.DataFrame, Indicator]]
 

qlib/backtest/utils.py

@@ -4,7 +4,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
-from typing import Any, Set, Tuple, TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Any, Set, Tuple, Union
 
 import numpy as np
 

qlib/cli/data.py

@@ -2,8 +2,8 @@
 #  Licensed under the MIT License.
 
 import fire
-from qlib.tests.data import GetData
 
+from qlib.tests.data import GetData
 
 if __name__ == "__main__":
     fire.Fire(GetData)

qlib/cli/run.py

@@ -2,8 +2,8 @@
 #  Licensed under the MIT License.
 import logging
 import os
-from pathlib import Path
 import sys
+from pathlib import Path
 
 import fire
 from jinja2 import Template, meta

qlib/config.py

@@ -12,17 +12,16 @@
 """
 from __future__ import annotations
 
-import os
-import re
 import copy
 import logging
-import platform
 import multiprocessing
+import os
+import platform
+import re
 from pathlib import Path
-from typing import Callable, Optional, Union
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable, Optional, Union
 
-from qlib.constant import REG_CN, REG_US, REG_TW
+from qlib.constant import REG_CN, REG_TW, REG_US
 
 if TYPE_CHECKING:
     from qlib.utils.time import Freq
@@ -438,7 +437,7 @@ def set(self, default_conf: str = "client", **kwargs):
         default_conf : str
             the default config template chosen by user: "server", "client"
         """
-        from .utils import set_log_with_config, get_module_logger, can_use_cache  # pylint: disable=C0415
+        from .utils import can_use_cache, get_module_logger, set_log_with_config  # pylint: disable=C0415
 
         self.reset()
 
@@ -480,10 +479,10 @@ def set(self, default_conf: str = "client", **kwargs):
                     )
 
     def register(self):
-        from .utils import init_instance_by_config  # pylint: disable=C0415
-        from .data.ops import register_all_ops  # pylint: disable=C0415
         from .data.data import register_all_wrappers  # pylint: disable=C0415
-        from .workflow import R, QlibRecorder  # pylint: disable=C0415
+        from .data.ops import register_all_ops  # pylint: disable=C0415
+        from .utils import init_instance_by_config  # pylint: disable=C0415
+        from .workflow import QlibRecorder, R  # pylint: disable=C0415
         from .workflow.utils import experiment_exit_handler  # pylint: disable=C0415
 
         register_all_ops(self)

qlib/contrib/data/data.py

@@ -7,11 +7,12 @@
 # - pip fail to computing the right version number!!!!
 #    - Maybe we can solve this problem by poetry
 
+import pandas as pd
+import pymongo
+
 # FIXME: So if you want to use arctic-based provider, please install arctic manually
 # `pip install arctic` may not be enough.
 from arctic import Arctic
-import pandas as pd
-import pymongo
 
 from qlib.data.data import FeatureProvider
 

qlib/contrib/data/dataset.py

@@ -2,15 +2,15 @@
 # Licensed under the MIT License.
 
 import copy
-import torch
 import warnings
+
 import numpy as np
 import pandas as pd
-from qlib.utils.data import guess_horizon
-from qlib.utils import init_instance_by_config
+import torch
 
 from qlib.data.dataset import DatasetH
-
+from qlib.utils import init_instance_by_config
+from qlib.utils.data import guess_horizon
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 

qlib/contrib/data/handler.py

@@ -1,12 +1,14 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+from inspect import getfullargspec
+
 from qlib.contrib.data.loader import Alpha158DL, Alpha360DL
+
+from ...data.dataset import processor as processor_module
 from ...data.dataset.handler import DataHandlerLP
 from ...data.dataset.processor import Processor
 from ...utils import get_callable_kwargs
-from ...data.dataset import processor as processor_module
-from inspect import getfullargspec
 
 
 def check_transform_proc(proc_l, fit_start_time, fit_end_time):

qlib/contrib/data/highfreq_processor.py

@@ -1,10 +1,11 @@
 import os
+from typing import Dict
 
 import numpy as np
 import pandas as pd
+
 from qlib.data.dataset.processor import Processor
 from qlib.data.dataset.utils import fetch_df_by_index
-from typing import Dict
 
 
 class HighFreqTrans(Processor):

qlib/contrib/data/highfreq_provider.py

@@ -1,18 +1,19 @@
+import datetime
 import os
+import pickle as pkl
 import time
-import datetime
 from typing import Optional
 
+from joblib import Parallel, delayed
+
 import qlib
 from qlib import get_module_logger
-from qlib.data import D
 from qlib.config import REG_CN
-from qlib.utils import init_instance_by_config
-from qlib.data.dataset.handler import DataHandlerLP
+from qlib.contrib.ops.high_freq import BFillNan, Cut, Date, DayLast, FFillNan, IsInf, IsNull, Select, get_calendar_day
+from qlib.data import D
 from qlib.data.data import Cal
-from qlib.contrib.ops.high_freq import get_calendar_day, DayLast, FFillNan, BFillNan, Date, Select, IsNull, IsInf, Cut
-import pickle as pkl
-from joblib import Parallel, delayed
+from qlib.data.dataset.handler import DataHandlerLP
+from qlib.utils import init_instance_by_config
 
 
 class HighFreqProvider:

qlib/contrib/data/processor.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from ...log import TimeInspector
 from ...data.dataset.processor import Processor, get_group_columns
+from ...log import TimeInspector
 
 
 class ConfigSectionProcessor(Processor):

qlib/contrib/data/utils/sepdf.py

@@ -1,8 +1,9 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-import pandas as pd
 from typing import Dict, Iterable, Union
 
+import pandas as pd
+
 
 def align_index(df_dict, join):
     res = {}

qlib/contrib/eva/alpha.py

@@ -4,12 +4,14 @@
 The interface should be redesigned carefully in the future.
 """
 
-import pandas as pd
 from typing import Tuple
-from qlib import get_module_logger
-from qlib.utils.paral import complex_parallel, DelayedDict
+
+import pandas as pd
 from joblib import Parallel, delayed
 
+from qlib import get_module_logger
+from qlib.utils.paral import DelayedDict, complex_parallel
+
 
 def calc_long_short_prec(
     pred: pd.Series, label: pd.Series, date_col="datetime", quantile: float = 0.2, dropna=False, is_alpha=False

qlib/contrib/evaluate.py

@@ -1,25 +1,24 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from __future__ import division
-from __future__ import print_function
+from __future__ import division, print_function
+
+import warnings
+from typing import Literal, Union
 
 import numpy as np
 import pandas as pd
-import warnings
-from typing import Union, Literal
 
+from ..backtest import backtest as backtest_func
+from ..backtest import executor as _executor
+from ..backtest import get_exchange, position
+from ..config import C
+from ..data import D
+from ..data.dataset.utils import get_level_index
 from ..log import get_module_logger
+from ..strategy.base import BaseStrategy
 from ..utils import get_date_range
 from ..utils.resam import Freq
-from ..strategy.base import BaseStrategy
-from ..backtest import get_exchange, position, backtest as backtest_func, executor as _executor
-
-
-from ..data import D
-from ..config import C
-from ..data.dataset.utils import get_level_index
-
 
 logger = get_module_logger("Evaluate")
 

qlib/contrib/evaluate_portfolio.py

@@ -2,17 +2,16 @@
 # Licensed under the MIT License.
 
 
-from __future__ import division
-from __future__ import print_function
+from __future__ import division, print_function
+
+from collections import OrderedDict
 
 import numpy as np
 import pandas as pd
-from scipy.stats import spearmanr, pearsonr
+from scipy.stats import pearsonr, spearmanr
 
 from ..data import D
 
-from collections import OrderedDict
-
 
 def _get_position_value_from_df(evaluate_date, position, close_data_df):
     """Get position value by existed close data df

qlib/contrib/meta/__init__.py

@@ -1,7 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from .data_selection import MetaTaskDS, MetaDatasetDS, MetaModelDS
-
+from .data_selection import MetaDatasetDS, MetaModelDS, MetaTaskDS
 
 __all__ = ["MetaTaskDS", "MetaDatasetDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/__init__.py

@@ -4,5 +4,4 @@
 from .dataset import MetaDatasetDS, MetaTaskDS
 from .model import MetaModelDS
 
-
 __all__ = ["MetaDatasetDS", "MetaTaskDS", "MetaModelDS"]