历史行情数据

1import os
2import shutil
3import urllib.request
4from datetime import datetime, timedelta
5
6BASE_URL = "https://download.gatedata.org"
7HOURLY_TYPES = {"orderbooks", "orderbooks_slice"}
8
9
10def build_url(biz: str, type_: str, market: str, dt_utc: datetime) -> str:
11    ym   = dt_utc.strftime("%Y%m")      # folder:  202304
12    ymdh = dt_utc.strftime("%Y%m%d%H")  # hourly:  2023042505
13
14    if type_ == "orderbooks_slice":
15        fname = f"{market}-{ymdh}.gz"
16    elif type_ == "orderbooks":
17        fname = f"{market}-{ymdh}.csv.gz"
18    else:
19        fname = f"{market}-{ym}.csv.gz"
20
21    return f"{BASE_URL}/{biz}/{type_}/{ym}/{fname}"
22
23
24def download_and_extract(url: str, save_dir: str) -> str:
25    # build sub-folder from URL: {save_dir}/{biz}/{type}
26    parts = url.split("/")
27    save_dir = os.path.join(save_dir, parts[3], parts[4])
28    os.makedirs(save_dir, exist_ok=True)
29    gz_name = parts[-1]
30    gz_path = os.path.join(save_dir, gz_name)
31
32    print(f"↓ {url}")
33    urllib.request.urlretrieve(url, gz_path)
34
35    # orderbooks_slice → .txt, others (.csv.gz) → .csv
36    if gz_name.endswith(".csv.gz"):
37        out_path = gz_path[: -len(".csv.gz")] + ".csv"
38    else:
39        out_path = gz_path[: -len(".gz")] + ".txt"
40
41    with gzip.open(gz_path, "rb") as f_in, open(out_path, "wb") as f_out:
42        shutil.copyfileobj(f_in, f_out)
43
44    os.remove(gz_path)
45    print(f"  saved → {out_path}")
46    return out_path
47
48
49def batch_download(biz: str, type_: str, market: str,
50                   save_dir: str,
51                   start_dt: datetime,
52                   end_dt: datetime = None):
53    """
54    If end_dt is omitted, downloads only the single file for start_dt.
55
56    Monthly granularity types: iterates month by month.
57      e.g. start_dt=datetime(2026,1,1), end_dt=datetime(2026,3,1)
58           → downloads 202601 / 202602 / 202603
59
60    Hourly granularity types (orderbooks / orderbooks_slice): iterates hour by hour.
61      e.g. start_dt=datetime(2026,1,1,0), end_dt=datetime(2026,1,2,23)
62           → downloads 2026010100 ~ 2026010223, 48 files in total
63    """
64    if end_dt is None:
65        end_dt = start_dt
66
67    if type_ in HOURLY_TYPES:
68        dt = start_dt
69        while dt <= end_dt:
70            download_and_extract(build_url(biz, type_, market, dt), save_dir)
71            dt += timedelta(hours=1)
72    else:
73        dt = start_dt.replace(day=1, hour=0, minute=0, second=0)
74        end = end_dt.replace(day=1, hour=0, minute=0, second=0)
75        while dt <= end:
76            download_and_extract(build_url(biz, type_, market, dt), save_dir)
77            dt = (dt.replace(day=1) + timedelta(days=32)).replace(day=1)
78
79
80if __name__ == "__main__":
81    # ── Spot ──────────────────────────────────────────────────────────────
82    # Single download
83    batch_download("spot", "deals",            "BTC_USDT", "./data", datetime(2023, 4, 25))
84    # batch_download("spot", "candlesticks_1m",  "BTC_USDT", "./data", datetime(2023, 4, 25))
85    # batch_download("spot", "candlesticks_5m",  "BTC_USDT", "./data", datetime(2023, 4, 25))
86    # batch_download("spot", "candlesticks_1h",  "BTC_USDT", "./data", datetime(2023, 4, 25))
87    # batch_download("spot", "candlesticks_4h",  "BTC_USDT", "./data", datetime(2023, 4, 25))
88    # batch_download("spot", "candlesticks_1d",  "BTC_USDT", "./data", datetime(2023, 4, 25))
89    # batch_download("spot", "candlesticks_7d",  "BTC_USDT", "./data", datetime(2023, 4, 25))
90    # batch_download("spot", "orderbooks",       "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
91    # batch_download("spot", "orderbooks_slice", "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
92    # Batch download
93    batch_download("spot", "deals",            "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
94    # batch_download("spot", "candlesticks_1m",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
95    # batch_download("spot", "candlesticks_5m",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
96    # batch_download("spot", "candlesticks_1h",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
97    # batch_download("spot", "candlesticks_4h",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
98    # batch_download("spot", "candlesticks_1d",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
99    # batch_download("spot", "candlesticks_7d",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
100    # batch_download("spot", "orderbooks",       "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
101    # batch_download("spot", "orderbooks_slice", "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
102
103    # ── futures_usdt ──────────────────────────────────────────────────────────
104    # Single download
105    # batch_download("futures_usdt", "trades",            "BTC_USDT", "./data", datetime(2023, 4, 25))
106    # batch_download("futures_usdt", "candlesticks_10s",  "BTC_USDT", "./data", datetime(2023, 4, 25))
107    # batch_download("futures_usdt", "candlesticks_1m",   "BTC_USDT", "./data", datetime(2023, 4, 25))
108    # batch_download("futures_usdt", "candlesticks_1h",   "BTC_USDT", "./data", datetime(2023, 4, 25))
109    # batch_download("futures_usdt", "candlesticks_4h",   "BTC_USDT", "./data", datetime(2023, 4, 25))
110    # batch_download("futures_usdt", "candlesticks_1d",   "BTC_USDT", "./data", datetime(2023, 4, 25))
111    # batch_download("futures_usdt", "candlesticks_7d",   "BTC_USDT", "./data", datetime(2023, 4, 25))
112    # batch_download("futures_usdt", "orderbooks",        "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
113    # batch_download("futures_usdt", "orderbooks_slice",  "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
114    # batch_download("futures_usdt", "mark_prices",       "BTC_USDT", "./data", datetime(2023, 4, 25))
115    # batch_download("futures_usdt", "funding_applies",   "BTC_USDT", "./data", datetime(2023, 4, 25))
116    # batch_download("futures_usdt", "funding_updates",   "BTC_USDT", "./data", datetime(2023, 4, 25))
117    # Batch download
118    # batch_download("futures_usdt", "trades",            "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
119    # batch_download("futures_usdt", "candlesticks_10s",  "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
120    # batch_download("futures_usdt", "candlesticks_1m",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
121    # batch_download("futures_usdt", "candlesticks_1h",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
122    # batch_download("futures_usdt", "candlesticks_4h",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
123    # batch_download("futures_usdt", "candlesticks_1d",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
124    # batch_download("futures_usdt", "candlesticks_7d",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
125    # batch_download("futures_usdt", "orderbooks",        "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
126    # batch_download("futures_usdt", "orderbooks_slice",  "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
127    # batch_download("futures_usdt", "mark_prices",       "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
128    # batch_download("futures_usdt", "funding_applies",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
129    # batch_download("futures_usdt", "funding_updates",   "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
130
131    # ── futures_btc ────────────────────────────────────────────────────────
132    # Single download
133    # batch_download("futures_btc", "trades",            "BTC_USD", "./data", datetime(2023, 4, 25))
134    # batch_download("futures_btc", "candlesticks_10s",  "BTC_USD", "./data", datetime(2023, 4, 25))
135    # batch_download("futures_btc", "candlesticks_1m",   "BTC_USD", "./data", datetime(2023, 4, 25))
136    # batch_download("futures_btc", "candlesticks_1h",   "BTC_USD", "./data", datetime(2023, 4, 25))
137    # batch_download("futures_btc", "candlesticks_4h",   "BTC_USD", "./data", datetime(2023, 4, 25))
138    # batch_download("futures_btc", "candlesticks_1d",   "BTC_USD", "./data", datetime(2023, 4, 25))
139    # batch_download("futures_btc", "candlesticks_7d",   "BTC_USD", "./data", datetime(2023, 4, 25))
140    # batch_download("futures_btc", "orderbooks",        "BTC_USD", "./data", datetime(2023, 4, 25, 5))
141    # batch_download("futures_btc", "orderbooks_slice",  "BTC_USD", "./data", datetime(2023, 4, 25, 5))
142    # batch_download("futures_btc", "mark_prices",       "BTC_USD", "./data", datetime(2023, 4, 25))
143    # batch_download("futures_btc", "funding_applies",   "BTC_USD", "./data", datetime(2023, 4, 25))
144    # batch_download("futures_btc", "funding_updates",   "BTC_USD", "./data", datetime(2023, 4, 25))
145    # Batch download
146    # batch_download("futures_btc", "trades",            "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
147    # batch_download("futures_btc", "candlesticks_10s",  "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
148    # batch_download("futures_btc", "candlesticks_1m",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
149    # batch_download("futures_btc", "candlesticks_1h",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
150    # batch_download("futures_btc", "candlesticks_4h",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
151    # batch_download("futures_btc", "candlesticks_1d",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
152    # batch_download("futures_btc", "candlesticks_7d",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
153    # batch_download("futures_btc", "orderbooks",        "BTC_USD", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
154    # batch_download("futures_btc", "orderbooks_slice",  "BTC_USD", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
155    # batch_download("futures_btc", "mark_prices",       "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
156    # batch_download("futures_btc", "funding_applies",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
157    # batch_download("futures_btc", "funding_updates",   "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))