提升数据准确性，优化部分函数性能

Author: Micro-sheep

changelog.md

@@ -1,48 +1,74 @@
 # Changelog
 
+## v0.5.5(2025-03-15)
+
+### Fixed
+
+- `get_realtime_quotes` 盘中数据存在重复数据
+
+### Changed
+
+- `get_all_company_performance` 和 `get_latest_holder_number`  速度优化
+
+---
+
 ## v0.5.4(2025-03-10)
 
 ### Fixed
 
 - `get_realtime_quotes` 分页更加智能
 
+---
+
 ## v0.5.3(2025-02-17)
 
 ### Fixed
 
 - `get_realtime_quotes` 自动分页以获取完整数据
 
+---
+
 ## v0.5.2(2024-04-24)
 
 ### Fixed
 
 - 补充 setup.py 缺失的依赖
 
+---
+
 ## v0.5.1(2024-02-22)
 
 ### Added
 
 - 增加用于检索股票市场的参数
 
+---
+
 ## v0.5.0(2023-01-08)
 
 ### Added
 
 - 增加获取 广期所 期货行情的功能
 - 添加扩展市场类型的函数
 
+---
+
 ## v0.4.9(2022-07-29)
 
 ### Added
 
 - 为 `get_quote_history` 添加更多选项
 
+---
+
 ## v0.4.8(2022-06-30)
 
 ### Fixed
 
 - 修复 `stock` 模块的 `get_base_info` 函数的 bug
 
+---
+
 ## v0.4.7(2022-06-24)
 
 ### Added
@@ -53,13 +79,17 @@
 
 - 修复部分函数命名拼写错误
 
+---
+
 ## v0.4.6(2022-06-19)
 
 ### Changed
 
 - 从 `stock` 模块中抽离出获取证券最新行情的函数
 - 增强 `common` 模块中的配置以支持自定义获取更多属性
 
+---
+
 ## v0.4.5(2022-06-05)
 
 ### Fixed
@@ -70,6 +100,8 @@
 
 为 `futures` 和 `bond` 模块添加获取最新交易日成交明细的功能
 
+---
+
 ## v0.4.4(2022-04-29)
 
 ### Added
@@ -80,13 +112,17 @@
 
 - 将 `stock` 模块里面的 `get_belong_plate` 重命名为 `get_belong_board`
 
+---
+
 ## v0.4.3(2022-04-28)
 
 ### Added
 
 - 为 `stock` 模块添加 `get_belong_plate` 函数，以支持获取股票所属板块
 - 为 `stock` 模块添加 `get_deal_detail` 函数，以支持获取股票最新交易日的成交明细
 
+---
+
 ## v0.4.2(2022-03-07)
 
 ### Changed
@@ -139,6 +175,8 @@
 - 添加获取多个板块成分股实时行情以及板块历史行情的功能
 - 添加获取 ETF、LOF 基金实时行情的功能
 
+---
+
 ## v0.3.7(2021-08-30)
 
 ### Added
@@ -147,6 +185,8 @@
 - 添加获取沪深 A 股股东数量的功能
 - 为 `stock` 模块添加龙虎榜数据获取功能
 
+---
+
 ### Fixed
 
 - 修复 `fund` 模块获取基金代码的函数产生的 bug

efinance/__version__.py

@@ -1,5 +1,5 @@
 __title__ = "efinance"
-__version__ = "0.5.4"
+__version__ = "0.5.5"
 __author__ = "micro sheep"
 __url__ = "https://github.com/Micro-sheep/efinance"
 __author_email__ = "[email protected]"

efinance/common/getter.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from typing import Dict, List, Union
-
+import numpy as np
 import multitasking
 import pandas as pd
 from jsonpath import jsonpath
@@ -40,13 +40,14 @@ def get_realtime_quotes_by_fs(fs: str, **kwargs) -> pd.DataFrame:
 
     def get_by_page(pn: int, pz: int):
         params = (
-            ("pn", f"{pn}"),
-            ("pz", f"{pz}"),
+            ("pn", pn),
+            ("pz", pz),
             ("po", "1"),
             ("np", "1"),
             ("fltt", "2"),
             ("invt", "2"),
-            ("fid", "f3"),
+            # NOTE 按代码排序。避免多次请求顺序不一致
+            ("fid", "f12"),
             ("fs", fs),
             ("fields", fields),
         )
@@ -56,6 +57,17 @@ def get_by_page(pn: int, pz: int):
         ).json()
         return json_response
 
+    def mixed_sort_key(series: pd.Series):
+        sort_keys = pd.Series(index=series.index, dtype=float)
+
+        for idx, val in series.items():
+            if isinstance(val, str):
+                sort_keys[idx] = np.nan
+            else:
+                sort_keys[idx] = val
+
+        return sort_keys
+
     json_response = get_by_page(1, pz=200)
     total = json_response["data"]["total"]
     pz = len(json_response["data"]["diff"])
@@ -70,9 +82,17 @@ def get_by_page(pn: int, pz: int):
         pd.DataFrame(response["data"]["diff"])[list(columns.keys())]
         for response in responses
     ]
-    df = pd.concat(dfs, axis=0, ignore_index=True).rename(columns=columns)[
-        columns.values()
-    ]
+    df = (
+        pd.concat(dfs, axis=0, ignore_index=True)
+        .rename(columns=columns)[columns.values()]
+        .sort_values(
+            by="涨跌幅",
+            ascending=False,
+            ignore_index=True,
+            key=mixed_sort_key,
+        )
+    )
+
     df["行情ID"] = df["市场编号"].astype(str) + "." + df["代码"].astype(str)
     df["市场类型"] = (
         df["市场编号"].astype(str).apply(lambda x: MARKET_NUMBER_DICT.get(x))

efinance/stock/getter.py

@@ -4,6 +4,7 @@
 import sys
 from datetime import datetime, timedelta
 from typing import Dict, List, Union
+from concurrent.futures import ThreadPoolExecutor
 
 import threading
 import multitasking
@@ -827,28 +828,37 @@ def get_all_company_performance(date: str = None) -> pd.DataFrame:
         return pd.DataFrame(columns=fields.values())
 
     date = f"(REPORTDATE='{date}')"
-    page = 1
-    dfs: List[pd.DataFrame] = []
-    while 1:
+
+    def get_by_page(pn: int, pz: int):
         params = (
             ("st", "NOTICE_DATE,SECURITY_CODE"),
             ("sr", "-1,-1"),
-            ("ps", "500"),
-            ("p", f"{page}"),
+            ("ps", pz),
+            ("p", pn),
             ("type", "RPT_LICO_FN_CPD"),
             ("sty", "ALL"),
             ("token", "894050c76af8597a853f5b408b759f5d"),
             # ! 只选沪深A股
             ("filter", f'(SECURITY_TYPE_CODE in ("058001001","058001008")){date}'),
         )
         url = "http://datacenter-web.eastmoney.com/api/data/get"
-        response = session.get(url, headers=EASTMONEY_REQUEST_HEADERS, params=params)
-        items = jsonpath(response.json(), "$..data[:]")
-        if not items:
-            break
-        df = pd.DataFrame(items)
-        dfs.append(df)
-        page += 1
+        json_response = session.get(
+            url, headers=EASTMONEY_REQUEST_HEADERS, params=params
+        ).json()
+        return json_response
+
+    json_response = get_by_page(1, pz=500)
+    total = json_response["result"]["count"]
+    pz = len(jsonpath(json_response, "$..data[:]"))
+    div, mod = divmod(total, pz)
+    pages = div + 1 if mod else div
+
+    with ThreadPoolExecutor() as executor:
+        tasks = executor.map(get_by_page, range(1, pages + 1), [pz] * pages)
+        responses = list(tasks)
+
+    dfs = [pd.DataFrame(jsonpath(response, "$..data[:]")) for response in responses]
+
     if len(dfs) == 0:
         df = pd.DataFrame(columns=fields.values())
         return df
@@ -940,12 +950,12 @@ def get_latest_holder_number(date: str = None) -> pd.DataFrame:
         "HOLD_NOTICE_DATE": "公告日期",
     }
 
-    while 1:
+    def get_by_page(pn: int, pz: int):
         params = [
             ("sortColumns", "HOLD_NOTICE_DATE,SECURITY_CODE"),
             ("sortTypes", "-1,-1"),
-            ("pageSize", "500"),
-            ("pageNumber", page),
+            ("pageSize", pz),
+            ("pageNumber", pn),
             (
                 "columns",
                 "SECURITY_CODE,SECURITY_NAME_ABBR,END_DATE,INTERVAL_CHRATE,AVG_MARKET_CAP,AVG_HOLD_NUM,TOTAL_MARKET_CAP,TOTAL_A_SHARES,HOLD_NOTICE_DATE,HOLDER_NUM,PRE_HOLDER_NUM,HOLDER_NUM_CHANGE,HOLDER_NUM_RATIO,END_DATE,PRE_END_DATE",
@@ -963,18 +973,26 @@ def get_latest_holder_number(date: str = None) -> pd.DataFrame:
 
         params = tuple(params)
         url = "http://datacenter-web.eastmoney.com/api/data/v1/get"
-        response = session.get(url, headers=EASTMONEY_REQUEST_HEADERS, params=params)
-        items = jsonpath(response.json(), "$..data[:]")
-        if not items:
-            break
-        df = pd.DataFrame(items)
-        df = df.rename(columns=fields)[fields.values()]
-        page += 1
-        dfs.append(df)
-    if len(dfs) == 0:
-        df = pd.DataFrame(columns=fields.values())
-        return df
-    df = pd.concat(dfs, ignore_index=True)
+        json_response = session.get(
+            url, headers=EASTMONEY_REQUEST_HEADERS, params=params
+        ).json()
+        return json_response
+
+    json_response = get_by_page(1, pz=500)
+    total = json_response["result"]["count"]
+    pz = len(jsonpath(json_response, "$..data[:]"))
+    div, mod = divmod(total, pz)
+    pages = div + 1 if mod else div
+
+    if total == 0:
+        return pd.DataFrame(columns=fields.values())
+
+    with ThreadPoolExecutor() as executor:
+        tasks = executor.map(get_by_page, range(1, pages + 1), [pz] * pages)
+        responses = list(tasks)
+
+    dfs = [pd.DataFrame(jsonpath(response, "$..data[:]")) for response in responses]
+    df = pd.concat(dfs, ignore_index=True).rename(columns=fields)[fields.values()]
     return df