How CaptureOCR achieves pixel-perfect captures across scaled and mixed-DPI displays
The Core Problem
Windows captures in physical pixels; Qt works in logical coordinates. On a 150% display, 100 logical pixels = 150 physical pixels. On mixed-DPI multi-monitor setups (e.g., 100% laptop + 150% external), each monitor has its own scale factor.
If you ignore this:
- Captures are blurry (logical rect scaled globally)
- Selections are offset (window frame not accounted for)
- Results attach to wrong thumbnail after reordering
Product link: CaptureOCR on Microsoft Store — $2.99 one-time, DPI-aware from day one.
Screen Representation
from dataclasses import dataclass
from PySide6.QtCore import QRect
@dataclass
class ScreenInfo:
index: int # Qt screen index
logical_rect: QRect # In Qt logical coordinates
physical_rect: QRect # In physical pixels (for mss)
dpr: float # Device pixel ratio (1.0, 1.25, 1.5, 2.0...)
@property
def name(self) -> str:
return f"Screen {self.index} ({self.dpr*100:.0f}%)"
Building the screen list at startup:
def build_screen_infos(app: QApplication) -> list[ScreenInfo]:
screens = []
for i, screen in enumerate(app.screens()):
logical = screen.geometry()
dpr = screen.devicePixelRatio()
physical = QRect(
int(logical.x() * dpr),
int(logical.y() * dpr),
int(logical.width() * dpr),
int(logical.height() * dpr)
)
screens.append(ScreenInfo(i, logical, physical, dpr))
return screens
Selection → Capture Pipeline
1. User drags selection in logical coordinates
The full-screen overlay works entirely in Qt logical space. User sees a rectangle in logical pixels.
2. Determine which monitor(s) the selection spans
def get_screens_for_rect(screens: list[ScreenInfo], logical_rect: QRect) -> list[ScreenInfo]:
"""Return screens intersected by the logical rect."""
intersected = []
for screen in screens:
if screen.logical_rect.intersects(logical_rect):
intersected.append(screen)
return intersected
3. Convert to physical rect per monitor
def logical_to_physical_per_screen(screen: ScreenInfo, logical_rect: QRect) -> QRect:
"""Convert logical rect to physical pixels for THIS monitor."""
# Origin relative to screen's logical origin
rel_x = logical_rect.x() - screen.logical_rect.x()
rel_y = logical_rect.y() - screen.logical_rect.y()
# Scale by THIS screen's DPR
phys_x = int(rel_x * screen.dpr)
phys_y = int(rel_y * screen.dpr)
phys_w = int(logical_rect.width() * screen.dpr)
phys_h = int(logical_rect.height() * screen.dpr)
# Absolute physical coordinates (for mss)
abs_x = screen.physical_rect.x() + phys_x
abs_y = screen.physical_rect.y() + phys_y
return QRect(abs_x, abs_y, phys_w, phys_h)
4. Capture with mss using physical coordinates
import mss
def capture_physical_rect(physical_rect: QRect) -> np.ndarray:
"""Grab exact physical pixel region using mss."""
with mss.mss() as sct:
monitor = {
"left": physical_rect.x(),
"top": physical_rect.y(),
"width": physical_rect.width(),
"height": physical_rect.height(),
}
img = sct.grab(monitor)
# mss returns BGRA, convert to BGR for OpenCV
return np.array(img)[:, :, :3]
Handling Cross-Monitor Selections
A selection can span two monitors with different DPRs.
def capture_selection(selection_logical: QRect, screens: list[ScreenInfo]) -> list[CaptureResult]:
"""
A selection spanning multiple monitors must be split and captured
per monitor at each monitor's native DPR.
"""
results = []
intersected = get_screens_for_rect(screens, selection_logical)
for screen in intersected:
# Intersection of selection with this screen (logical)
intersection = selection_logical.intersected(screen.logical_rect)
if intersection.isEmpty():
continue
# Convert to physical for THIS screen
physical = logical_to_physical_per_screen(screen, intersection)
# Capture
img = capture_physical_rect(physical)
results.append(CaptureResult(
image=img,
screen_index=screen.index,
logical_rect=intersection,
physical_rect=physical,
dpr=screen.dpr
))
return results
Result: Each monitor region captured at its native resolution. No global scaling artifacts.
Stable Region Identity vs. Display Order
Users expect regions numbered ① ② ③ in display order (top-to-bottom, left-to-right). But if they delete ② and move ⑤ up, the underlying identity must stay stable.
@dataclass
class Selection:
stable_id: int # Never changes (UUID or incrementing counter)
display_index: int # Current position in drawer (1-based)
logical_rect: QRect # In logical coordinates
screen_index: int # Which monitor
dpr: float # DPR at capture time
physical_rect: QRect # Physical pixels captured
image: np.ndarray # The captured pixels
ocr_result: OcrResult | None = None
Reordering logic:
class SelectionDrawer:
def __init__(self):
self.selections: list[Selection] = []
self.next_stable_id = 0
def add(self, logical_rect: QRect, screen_index: int, dpr: float, physical_rect: QRect, image: np.ndarray) -> Selection:
sel = Selection(
stable_id=self.next_stable_id,
display_index=len(self.selections) + 1,
logical_rect=logical_rect,
screen_index=screen_index,
dpr=dpr,
physical_rect=physical_rect,
image=image
)
self.next_stable_id += 1
self.selections.append(sel)
return sel
def remove(self, stable_id: int):
self.selections = [s for s in self.selections if s.stable_id != stable_id]
self._renumber_display_indices()
def move(self, stable_id: int, new_display_index: int):
sel = next(s for s in self.selections if s.stable_id == stable_id)
old_idx = sel.display_index
# Shift others
for s in self.selections:
if s.display_index == new_display_index:
s.display_index = old_idx
sel.display_index = new_display_index
self.selections.sort(key=lambda s: s.display_index)
def _renumber_display_indices(self):
for i, sel in enumerate(sorted(self.selections, key=lambda s: s.display_index)):
sel.display_index = i + 1
Export uses display order — that's what humans expect in a table.
Window Frame Offset Bug
On Windows, a logical rect from Qt includes the window frame if you're not careful.
def get_client_logical_rect(window: QWidget) -> QRect:
"""Get the client area (content) in logical coordinates."""
# mapToGlobal(0,0) gives top-left of content (excludes frame)
top_left = window.mapToGlobal(QPoint(0, 0))
bottom_right = window.mapToGlobal(QPoint(window.width(), window.height()))
return QRect(top_left, bottom_right)
Always capture client area, not window frame.
Testing Matrix
| Scenario | Expected Behavior |
|---|---|
| 100% single monitor | 1:1 logical=physical |
| 150% single monitor | Logical ×1.5 = physical |
| 100% + 150% dual | Each monitor at native DPR |
| 200% + 100% dual | High-DPI captures sharp, low-DPI normal |
| Cross-monitor selection | Split per monitor, each at native DPR |
| Window dragged across monitors | Recompute screen_index on move |
Performance
- mss grab: ~2-5ms per capture (depends on resolution)
- DPI conversion: Negligible (<0.1ms)
- Memory: Each capture = W×H×3 bytes (BGR)
Key Takeaways
-
Never assume 1:1 logical:physical — query
devicePixelRatio()per screen - Capture per monitor at native DPR — don't scale globally
- Stable IDs + display indices — identity vs. presentation
- Test on real scaled hardware — 125%, 150%, 200%, mixed-DPI
- mss + Qt coordinate mapping — the combo that works
Resources
- mss docs: python-mss.readthedocs.io
- Qt High DPI: doc.qt.io/qt-6/highdpi.html
- CaptureOCR: Microsoft Store
Top comments (0)