mBA-Terminal / Python /mt5_filled_ticks.py

Upload 29 files

c99df4c verified 8 months ago

10.1 kB

"""
Script 2 — Gap-Filled Microsecond Bid-Ask Unit Data Visualization
Fetches the SAME XAUUSDc data as Script 1, then fills every missing
price level between consecutive data points so that the Y-distribution
histogram reflects the full path traversed, not just the endpoints.

Output: 4-panel figure identical in layout to Script 1 but built on the
gap-filled DataFrame.

0.01 unit = $0.01 XAU price change.
The 'c' suffix in XAUUSDc is an Exness broker account-type indicator
(standard cent live account), not related to XAU pricing.
"""

import MetaTrader5 as mt5
import pandas as pd
import numpy as np
import matplotlib
matplotlib.use('Agg')  # Headless backend — no GUI window
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from datetime import datetime, timezone

# ──────────────────────────────────────────────
# 1. Connect to MT5
# ──────────────────────────────────────────────
if not mt5.initialize():
    print(f"MT5 initialize() failed, error code = {mt5.last_error()}")
    quit()

# ──────────────────────────────────────────────
# 2. Define time range (Feb 12 2026, full day UTC)
# ──────────────────────────────────────────────
utc_from = datetime(2026, 2, 12, 0, 0, 0, tzinfo=timezone.utc)
utc_to   = datetime(2026, 2, 12, 23, 59, 59, tzinfo=timezone.utc)

SYMBOL    = "XAUUSDc"
UNIT_SIZE = 0.01  # the binsize (0.01 unit = $0.01 XAU price change)

# ──────────────────────────────────────────────
# 3. Fetch data from MT5 (same query as Script 1)
# ──────────────────────────────────────────────
ticks = mt5.copy_ticks_range(SYMBOL, utc_from, utc_to, mt5.COPY_TICKS_ALL)

if ticks is None or len(ticks) == 0:
    print(f"No data retrieved for {SYMBOL}. Error: {mt5.last_error()}")
    mt5.shutdown()
    quit()

df = pd.DataFrame(ticks)
df['datetime'] = pd.to_datetime(df['time_msc'], unit='ms', utc=True)

print(f"Fetched {len(df):,} raw data points for {SYMBOL}")
mt5.shutdown()

# Save raw unit CSV (same data as Script 1)
csv_raw = "raw_ticks_XAUUSDc_20260212.csv"  # filename kept for compatibility
df[['datetime', 'bid', 'ask', 'last', 'volume', 'flags']].to_csv(csv_raw, index=False)
print(f"Saved CSV → {csv_raw}  ({len(df):,} rows)")

# ──────────────────────────────────────────────
# 4. Vectorised gap-filling function
# ──────────────────────────────────────────────
def fill_gaps(prices: np.ndarray, timestamps_ns: np.ndarray, unit_size: float):
    """
    Vectorised gap-fill: for every consecutive pair (A → B),
    insert intermediate price levels at every unit_size step.
    timestamps_ns should be int64 nanoseconds.
    """
    diff_units = np.round(np.diff(prices) / unit_size).astype(np.int64)
    counts = np.abs(diff_units)
    # Last point gets a count of 1 (just itself)
    counts = np.append(counts, 1)

    total = int(np.sum(counts))
    indices = np.repeat(np.arange(len(prices)), counts)

    # Offset within each segment
    cum = np.cumsum(counts)
    starts = np.empty_like(cum)
    starts[0] = 0
    starts[1:] = cum[:-1]
    offsets = np.arange(total) - np.repeat(starts, counts)

    # Direction per segment
    directions = np.zeros(len(prices), dtype=np.float64)
    directions[:-1] = np.sign(diff_units)

    # Time step per segment
    dt = np.zeros(len(prices), dtype=np.float64)
    dt[:-1] = np.diff(timestamps_ns).astype(np.float64)
    steps = dt / np.where(counts > 0, counts, 1)

    filled_prices = prices[indices] + offsets * directions[indices] * unit_size
    filled_ts = timestamps_ns[indices].astype(np.float64) + offsets * steps[indices]

    return np.round(filled_prices, 2), filled_ts.astype(np.int64)


# ──────────────────────────────────────────────
# 5. Apply gap-filling to bid and ask separately
# ──────────────────────────────────────────────
ts_ns = df['datetime'].values.astype('datetime64[ns]').astype(np.int64)

print("Gap-filling bid...")
bid_prices_filled, bid_ts_filled = fill_gaps(df['bid'].values, ts_ns, UNIT_SIZE)
print("Gap-filling ask...")
ask_prices_filled, ask_ts_filled = fill_gaps(df['ask'].values, ts_ns, UNIT_SIZE)

print(f"Bid: {len(df):,} raw → {len(bid_prices_filled):,} filled rows")
print(f"Ask: {len(df):,} raw → {len(ask_prices_filled):,} filled rows")

# Save gap-filled CSVs
bid_filled_df = pd.DataFrame({
    'datetime': pd.to_datetime(bid_ts_filled, unit='ns', utc=True),
    'bid_filled': bid_prices_filled,
})
bid_csv = "filled_bid_XAUUSDc_20260212.csv"
bid_filled_df.to_csv(bid_csv, index=False)
print(f"Saved CSV → {bid_csv}  ({len(bid_filled_df):,} rows)")

ask_filled_df = pd.DataFrame({
    'datetime': pd.to_datetime(ask_ts_filled, unit='ns', utc=True),
    'ask_filled': ask_prices_filled,
})
ask_csv = "filled_ask_XAUUSDc_20260212.csv"
ask_filled_df.to_csv(ask_csv, index=False)
print(f"Saved CSV → {ask_csv}  ({len(ask_filled_df):,} rows)")

# Convert ns timestamps to matplotlib date floats
# matplotlib dates = days since 0001-01-01; Unix epoch = day 719163
_UNIX_EPOCH_MPLDATE = 719163.0
bid_times = bid_ts_filled / 1e9 / 86400.0 + _UNIX_EPOCH_MPLDATE
ask_times = ask_ts_filled / 1e9 / 86400.0 + _UNIX_EPOCH_MPLDATE

# ──────────────────────────────────────────────
# 6. Build histogram bins (1 bin = 0.01 unit)
# ──────────────────────────────────────────────
overall_min = min(bid_prices_filled.min(), ask_prices_filled.min())
overall_max = max(bid_prices_filled.max(), ask_prices_filled.max())

bin_lo = np.floor(overall_min / UNIT_SIZE) * UNIT_SIZE - UNIT_SIZE
bin_hi = np.ceil(overall_max / UNIT_SIZE) * UNIT_SIZE + UNIT_SIZE
bins = np.round(np.arange(bin_lo, bin_hi + UNIT_SIZE, UNIT_SIZE), 2)

print("Plotting...")

# ──────────────────────────────────────────────
# 7. Plot 4-panel figure
# ──────────────────────────────────────────────
fig, axes = plt.subplots(
    2, 2,
    figsize=(20, 12),
    gridspec_kw={'width_ratios': [1, 4]},
    sharey='row',
)
fig.suptitle(
    f'{SYMBOL} — Gap-Filled Unit Data (Path-Weighted)  |  {utc_from.strftime("%Y-%m-%d")}',
    fontsize=16, fontweight='bold',
)

# Colors — 100% blue and 100% red per IDEA.md
BID_COLOR = '#0000FF'
ASK_COLOR = '#FF0000'

# ── Row 0: BID ─────────────────────────────
ax_hist_bid = axes[0, 0]
ax_line_bid = axes[0, 1]

ax_hist_bid.hist(
    bid_prices_filled, bins=bins, orientation='horizontal',
    color=BID_COLOR, alpha=1.0, edgecolor='white', linewidth=0.3,
)
ax_hist_bid.set_xlabel('Count (path-weighted)', fontsize=10)
ax_hist_bid.set_ylabel('Bid Price', fontsize=10)
ax_hist_bid.set_title('Bid Y-Distribution — Gap-Filled (0.01-unit bins)', fontsize=12)
# histogram grows left-to-right (starts from 0)

# Line only — no markers for 4M+ points, rasterized
ax_line_bid.plot(
    bid_times, bid_prices_filled,
    color=BID_COLOR, linewidth=0.5, alpha=1.0,
    rasterized=True,
)
ax_line_bid.xaxis_date()
ax_line_bid.set_title('Bid Price — Gap-Filled (Time Series)', fontsize=12)
ax_line_bid.set_xlabel('Time (UTC)', fontsize=10)
ax_line_bid.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
ax_line_bid.xaxis.set_major_locator(mdates.HourLocator(interval=2))
plt.setp(ax_line_bid.xaxis.get_majorticklabels(), rotation=45, ha='right')
ax_line_bid.grid(True, alpha=0.3)

# ── Row 1: ASK ─────────────────────────────
ax_hist_ask = axes[1, 0]
ax_line_ask = axes[1, 1]

ax_hist_ask.hist(
    ask_prices_filled, bins=bins, orientation='horizontal',
    color=ASK_COLOR, alpha=1.0, edgecolor='white', linewidth=0.3,
)
ax_hist_ask.set_xlabel('Count (path-weighted)', fontsize=10)
ax_hist_ask.set_ylabel('Ask Price', fontsize=10)
ax_hist_ask.set_title('Ask Y-Distribution — Gap-Filled (0.01-unit bins)', fontsize=12)
# histogram grows left-to-right (starts from 0)

ax_line_ask.plot(
    ask_times, ask_prices_filled,
    color=ASK_COLOR, linewidth=0.5, alpha=1.0,
    rasterized=True,
)
ax_line_ask.xaxis_date()
ax_line_ask.set_title('Ask Price — Gap-Filled (Time Series)', fontsize=12)
ax_line_ask.set_xlabel('Time (UTC)', fontsize=10)
ax_line_ask.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
ax_line_ask.xaxis.set_major_locator(mdates.HourLocator(interval=2))
plt.setp(ax_line_ask.xaxis.get_majorticklabels(), rotation=45, ha='right')
ax_line_ask.grid(True, alpha=0.3)

# ── Final layout ───────────────────────────
plt.tight_layout(rect=[0, 0, 1, 0.95])

output_path = "filled_ticks_4panel.png"
fig.savefig(output_path, dpi=150, bbox_inches='tight')
print(f"Saved → {output_path}")

	"""
	Script 2 — Gap-Filled Microsecond Bid-Ask Unit Data Visualization
	Fetches the SAME XAUUSDc data as Script 1, then fills every missing
	price level between consecutive data points so that the Y-distribution
	histogram reflects the full path traversed, not just the endpoints.

	Output: 4-panel figure identical in layout to Script 1 but built on the
	gap-filled DataFrame.

	0.01 unit = $0.01 XAU price change.
	The 'c' suffix in XAUUSDc is an Exness broker account-type indicator
	(standard cent live account), not related to XAU pricing.
	"""

	import MetaTrader5 as mt5
	import pandas as pd
	import numpy as np
	import matplotlib
	matplotlib.use('Agg') # Headless backend — no GUI window
	import matplotlib.pyplot as plt
	import matplotlib.dates as mdates
	from datetime import datetime, timezone

	# ──────────────────────────────────────────────
	# 1. Connect to MT5
	# ──────────────────────────────────────────────
	if not mt5.initialize():
	print(f"MT5 initialize() failed, error code = {mt5.last_error()}")
	quit()

	# ──────────────────────────────────────────────
	# 2. Define time range (Feb 12 2026, full day UTC)
	# ──────────────────────────────────────────────
	utc_from = datetime(2026, 2, 12, 0, 0, 0, tzinfo=timezone.utc)
	utc_to = datetime(2026, 2, 12, 23, 59, 59, tzinfo=timezone.utc)

	SYMBOL = "XAUUSDc"
	UNIT_SIZE = 0.01 # the binsize (0.01 unit = $0.01 XAU price change)

	# ──────────────────────────────────────────────
	# 3. Fetch data from MT5 (same query as Script 1)
	# ──────────────────────────────────────────────
	ticks = mt5.copy_ticks_range(SYMBOL, utc_from, utc_to, mt5.COPY_TICKS_ALL)

	if ticks is None or len(ticks) == 0:
	print(f"No data retrieved for {SYMBOL}. Error: {mt5.last_error()}")
	mt5.shutdown()
	quit()

	df = pd.DataFrame(ticks)
	df['datetime'] = pd.to_datetime(df['time_msc'], unit='ms', utc=True)

	print(f"Fetched {len(df):,} raw data points for {SYMBOL}")
	mt5.shutdown()

	# Save raw unit CSV (same data as Script 1)
	csv_raw = "raw_ticks_XAUUSDc_20260212.csv" # filename kept for compatibility
	df[['datetime', 'bid', 'ask', 'last', 'volume', 'flags']].to_csv(csv_raw, index=False)
	print(f"Saved CSV → {csv_raw} ({len(df):,} rows)")

	# ──────────────────────────────────────────────
	# 4. Vectorised gap-filling function
	# ──────────────────────────────────────────────
	def fill_gaps(prices: np.ndarray, timestamps_ns: np.ndarray, unit_size: float):
	"""
	Vectorised gap-fill: for every consecutive pair (A → B),
	insert intermediate price levels at every unit_size step.
	timestamps_ns should be int64 nanoseconds.
	"""
	diff_units = np.round(np.diff(prices) / unit_size).astype(np.int64)
	counts = np.abs(diff_units)
	# Last point gets a count of 1 (just itself)
	counts = np.append(counts, 1)

	total = int(np.sum(counts))
	indices = np.repeat(np.arange(len(prices)), counts)

	# Offset within each segment
	cum = np.cumsum(counts)
	starts = np.empty_like(cum)
	starts[0] = 0
	starts[1:] = cum[:-1]
	offsets = np.arange(total) - np.repeat(starts, counts)

	# Direction per segment
	directions = np.zeros(len(prices), dtype=np.float64)
	directions[:-1] = np.sign(diff_units)

	# Time step per segment
	dt = np.zeros(len(prices), dtype=np.float64)
	dt[:-1] = np.diff(timestamps_ns).astype(np.float64)
	steps = dt / np.where(counts > 0, counts, 1)

	filled_prices = prices[indices] + offsets * directions[indices] * unit_size
	filled_ts = timestamps_ns[indices].astype(np.float64) + offsets * steps[indices]

	return np.round(filled_prices, 2), filled_ts.astype(np.int64)


	# ──────────────────────────────────────────────
	# 5. Apply gap-filling to bid and ask separately
	# ──────────────────────────────────────────────
	ts_ns = df['datetime'].values.astype('datetime64[ns]').astype(np.int64)

	print("Gap-filling bid...")
	bid_prices_filled, bid_ts_filled = fill_gaps(df['bid'].values, ts_ns, UNIT_SIZE)
	print("Gap-filling ask...")
	ask_prices_filled, ask_ts_filled = fill_gaps(df['ask'].values, ts_ns, UNIT_SIZE)

	print(f"Bid: {len(df):,} raw → {len(bid_prices_filled):,} filled rows")
	print(f"Ask: {len(df):,} raw → {len(ask_prices_filled):,} filled rows")

	# Save gap-filled CSVs
	bid_filled_df = pd.DataFrame({
	'datetime': pd.to_datetime(bid_ts_filled, unit='ns', utc=True),
	'bid_filled': bid_prices_filled,
	})
	bid_csv = "filled_bid_XAUUSDc_20260212.csv"
	bid_filled_df.to_csv(bid_csv, index=False)
	print(f"Saved CSV → {bid_csv} ({len(bid_filled_df):,} rows)")

	ask_filled_df = pd.DataFrame({
	'datetime': pd.to_datetime(ask_ts_filled, unit='ns', utc=True),
	'ask_filled': ask_prices_filled,
	})
	ask_csv = "filled_ask_XAUUSDc_20260212.csv"
	ask_filled_df.to_csv(ask_csv, index=False)
	print(f"Saved CSV → {ask_csv} ({len(ask_filled_df):,} rows)")

	# Convert ns timestamps to matplotlib date floats
	# matplotlib dates = days since 0001-01-01; Unix epoch = day 719163
	_UNIX_EPOCH_MPLDATE = 719163.0
	bid_times = bid_ts_filled / 1e9 / 86400.0 + _UNIX_EPOCH_MPLDATE
	ask_times = ask_ts_filled / 1e9 / 86400.0 + _UNIX_EPOCH_MPLDATE

	# ──────────────────────────────────────────────
	# 6. Build histogram bins (1 bin = 0.01 unit)
	# ──────────────────────────────────────────────
	overall_min = min(bid_prices_filled.min(), ask_prices_filled.min())
	overall_max = max(bid_prices_filled.max(), ask_prices_filled.max())

	bin_lo = np.floor(overall_min / UNIT_SIZE) * UNIT_SIZE - UNIT_SIZE
	bin_hi = np.ceil(overall_max / UNIT_SIZE) * UNIT_SIZE + UNIT_SIZE
	bins = np.round(np.arange(bin_lo, bin_hi + UNIT_SIZE, UNIT_SIZE), 2)

	print("Plotting...")

	# ──────────────────────────────────────────────
	# 7. Plot 4-panel figure
	# ──────────────────────────────────────────────
	fig, axes = plt.subplots(
	2, 2,
	figsize=(20, 12),
	gridspec_kw={'width_ratios': [1, 4]},
	sharey='row',
	)
	fig.suptitle(
	f'{SYMBOL} — Gap-Filled Unit Data (Path-Weighted) \| {utc_from.strftime("%Y-%m-%d")}',
	fontsize=16, fontweight='bold',
	)

	# Colors — 100% blue and 100% red per IDEA.md
	BID_COLOR = '#0000FF'
	ASK_COLOR = '#FF0000'

	# ── Row 0: BID ─────────────────────────────
	ax_hist_bid = axes[0, 0]
	ax_line_bid = axes[0, 1]

	ax_hist_bid.hist(
	bid_prices_filled, bins=bins, orientation='horizontal',
	color=BID_COLOR, alpha=1.0, edgecolor='white', linewidth=0.3,
	)
	ax_hist_bid.set_xlabel('Count (path-weighted)', fontsize=10)
	ax_hist_bid.set_ylabel('Bid Price', fontsize=10)
	ax_hist_bid.set_title('Bid Y-Distribution — Gap-Filled (0.01-unit bins)', fontsize=12)
	# histogram grows left-to-right (starts from 0)

	# Line only — no markers for 4M+ points, rasterized
	ax_line_bid.plot(
	bid_times, bid_prices_filled,
	color=BID_COLOR, linewidth=0.5, alpha=1.0,
	rasterized=True,
	)
	ax_line_bid.xaxis_date()
	ax_line_bid.set_title('Bid Price — Gap-Filled (Time Series)', fontsize=12)
	ax_line_bid.set_xlabel('Time (UTC)', fontsize=10)
	ax_line_bid.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
	ax_line_bid.xaxis.set_major_locator(mdates.HourLocator(interval=2))
	plt.setp(ax_line_bid.xaxis.get_majorticklabels(), rotation=45, ha='right')
	ax_line_bid.grid(True, alpha=0.3)

	# ── Row 1: ASK ─────────────────────────────
	ax_hist_ask = axes[1, 0]
	ax_line_ask = axes[1, 1]

	ax_hist_ask.hist(
	ask_prices_filled, bins=bins, orientation='horizontal',
	color=ASK_COLOR, alpha=1.0, edgecolor='white', linewidth=0.3,
	)
	ax_hist_ask.set_xlabel('Count (path-weighted)', fontsize=10)
	ax_hist_ask.set_ylabel('Ask Price', fontsize=10)
	ax_hist_ask.set_title('Ask Y-Distribution — Gap-Filled (0.01-unit bins)', fontsize=12)
	# histogram grows left-to-right (starts from 0)

	ax_line_ask.plot(
	ask_times, ask_prices_filled,
	color=ASK_COLOR, linewidth=0.5, alpha=1.0,
	rasterized=True,
	)
	ax_line_ask.xaxis_date()
	ax_line_ask.set_title('Ask Price — Gap-Filled (Time Series)', fontsize=12)
	ax_line_ask.set_xlabel('Time (UTC)', fontsize=10)
	ax_line_ask.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
	ax_line_ask.xaxis.set_major_locator(mdates.HourLocator(interval=2))
	plt.setp(ax_line_ask.xaxis.get_majorticklabels(), rotation=45, ha='right')
	ax_line_ask.grid(True, alpha=0.3)

	# ── Final layout ───────────────────────────
	plt.tight_layout(rect=[0, 0, 1, 0.95])

	output_path = "filled_ticks_4panel.png"
	fig.savefig(output_path, dpi=150, bbox_inches='tight')
	print(f"Saved → {output_path}")