diff --git a/README.md b/README.md index d6f82f8..9516ecc 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,14 @@ The text clipboard is shared both ways. An open-source take on DeskDock, for mac - Real Android pointer and keyboard: the device sees a USB mouse and keyboard (UHID). - Nothing to install on the device: the Mac starts a small server over adb. - Text clipboard sync in both directions. -- Works with several displays: only edges that don't lead to another display count. +- Works with several displays: arrange the device against one or two of them, as in System + Settings > Displays; only the stretches it touches lead to it. +- Follows the device when it turns, and foldables when they switch screens (e.g. a Galaxy Z Fold + closed, on its cover screen). +- Several Android devices on adb: pick the one to use in the menu. - English and Brazilian Portuguese. -Status: early (0.1). USB only for now; Wi-Fi and file drag-and-drop are planned. +Status: early (0.2). USB only for now; Wi-Fi and file drag-and-drop are planned. ## Requirements diff --git a/docs/architecture.md b/docs/architecture.md index 7309a82..b8861b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,7 +18,9 @@ The mouse is relative and Android accelerates it, so the Mac can't compute where is. The shell user can't monitor input events, and the system's cursor-position API is internal to `system_server`. What the shell user can do is ask the input service for a dump: it lists each device's hovering pointer in display coordinates, e.g. -`DeviceId(15):[... hoveringPointers=[Pointer(id=0, MOUSE) at (0, 1835.57)]]`. A dump takes +`DeviceId(15):[... hoveringPointers=[Pointer(id=0, MOUSE) at (0, 1835.57)]]`. Those +coordinates are the display's unrotated (physical) ones, while the frame next to them is the rotated +one, so the server turns them using the display orientation from the same dump. A dump takes about 20 ms on a recent phone, so the server only takes one (at most every 30 ms) while the pointer moves toward the edge that leads back to the Mac. Once the pointer touches that edge and keeps being pushed against it, the server tells the Mac, with the position along the edge. @@ -36,7 +38,9 @@ When the Mac cursor is pushed out of a free screen edge, the Mac freezes its cur (`CGAssociateMouseAndMouseCursorPosition(false)`), hides it and sends `ENTER` with the position along the edge. The server pushes the pointer into the corner of the entry edge, then walks it along the edge, measuring with the dump, until it is within about 12 px of the matching position -(0.3-0.5 s). From then on the Mac forwards raw mouse deltas (times the speed setting), buttons, +(0.1-0.2 s; the pixels-per-count gain of each axis is learned on earlier entries). When the +screen turns or a foldable switches screens, the server sends the new size again and the Mac +reshapes the device in its arrangement, keeping the sides that touch displays. From then on the Mac forwards raw mouse deltas (times the speed setting), buttons, scrolling and keys, and swallows them locally through a `CGEventTap`. ## The Mac side diff --git a/mac/Resources/en.lproj/Localizable.strings b/mac/Resources/en.lproj/Localizable.strings index 72a3bcf..348bcde 100644 --- a/mac/Resources/en.lproj/Localizable.strings +++ b/mac/Resources/en.lproj/Localizable.strings @@ -1,8 +1,8 @@ "status.noAdb" = "adb not found: install Android platform-tools"; "status.waiting" = "Waiting for an Android device (USB debugging)"; "status.connecting" = "Connecting…"; -"status.connected" = "Connected: %@"; -"status.onDevice" = "Controlling %@"; +"status.connected" = "Connected over %2$@: %1$@"; +"status.onDevice" = "Controlling %1$@ (%2$@)"; "status.failed" = "Could not connect: %@"; "menu.grantAccessibility" = "Allow Accessibility access…"; "menu.share" = "Share mouse and keyboard"; @@ -31,3 +31,7 @@ "passage.drop" = "Drop to attach to %1$@: %2$@ edge, from %3$d%% to %4$d%% of its %5$@."; "passage.enters" = "The pointer enters the Android device through the side that touches the display, at the matching position."; "passage.dropNone" = "Not touching any display: dropping it here puts it back where it was."; +"status.chooseDevice" = "Several Android devices: choose one in Device"; +"transport.usb" = "USB"; +"transport.wifi" = "Wi-Fi"; +"menu.device" = "Device"; diff --git a/mac/Resources/pt-BR.lproj/Localizable.strings b/mac/Resources/pt-BR.lproj/Localizable.strings index f2cf71c..461f755 100644 --- a/mac/Resources/pt-BR.lproj/Localizable.strings +++ b/mac/Resources/pt-BR.lproj/Localizable.strings @@ -1,8 +1,8 @@ "status.noAdb" = "adb não encontrado: instale o Android platform-tools"; "status.waiting" = "Aguardando um Android (depuração USB)"; "status.connecting" = "Conectando…"; -"status.connected" = "Conectado: %@"; -"status.onDevice" = "Controlando %@"; +"status.connected" = "Conectado por %2$@: %1$@"; +"status.onDevice" = "Controlando %1$@ (%2$@)"; "status.failed" = "Não foi possível conectar: %@"; "menu.grantAccessibility" = "Permitir acesso de Acessibilidade…"; "menu.share" = "Compartilhar mouse e teclado"; @@ -31,3 +31,7 @@ "passage.drop" = "Solte para encostar em %1$@: borda %2$@, de %3$d%% a %4$d%% da %5$@."; "passage.enters" = "O ponteiro entra no Android pelo lado que encosta no monitor, na posição correspondente."; "passage.dropNone" = "Longe dos monitores: soltando aqui, ele volta para onde estava."; +"status.chooseDevice" = "Vários Androids conectados: escolha um em Aparelho"; +"transport.usb" = "USB"; +"transport.wifi" = "Wi-Fi"; +"menu.device" = "Aparelho"; diff --git a/mac/Sources/DroidBridge/AppDelegate.swift b/mac/Sources/DroidBridge/AppDelegate.swift index 8cc5e4a..540e2ac 100644 --- a/mac/Sources/DroidBridge/AppDelegate.swift +++ b/mac/Sources/DroidBridge/AppDelegate.swift @@ -145,6 +145,20 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { share.state = Settings.enabled ? .on : .off menu.addItem(share) + if link.devices.count > 1 || Settings.deviceSerial != nil { + let devices = NSMenuItem(title: L("menu.device"), action: nil, keyEquivalent: "") + let deviceMenu = NSMenu() + for d in link.devices { + let via = [d.usbSerial != nil ? L("transport.usb") : nil, d.wirelessSerial != nil ? L("transport.wifi") : nil] + .compactMap { $0 }.joined(separator: " + ") + let i = item("\(d.model) (\(via))", #selector(chooseDevice(_:))) + i.representedObject = d.id + i.state = (Settings.deviceSerial ?? (link.devices.count == 1 ? d.id : nil)) == d.id ? .on : .off + deviceMenu.addItem(i) + } + devices.submenu = deviceMenu + menu.addItem(devices) + } menu.addItem(item(L("menu.arrange"), #selector(openArrangement))) let speed = NSMenuItem(title: L("menu.speed"), action: nil, keyEquivalent: "") @@ -177,7 +191,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { case .waiting: return L("status.waiting") case .connecting: return L("status.connecting") case let .connected(model, _, _): - return String(format: L(capture.isRemote ? "status.onDevice" : "status.connected"), model) + let key = capture.isRemote ? "status.onDevice" : "status.connected" + return String(format: L(key), model, L(link.transport == .usb ? "transport.usb" : "transport.wifi")) + case .chooseDevice: return L("status.chooseDevice") case let .failed(reason): return String(format: L("status.failed"), reason) } } @@ -199,6 +215,12 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { if !Settings.enabled { capture.returnToMac() } } + @objc private func chooseDevice(_ sender: NSMenuItem) { + guard let id = sender.representedObject as? String else { return } + capture.returnToMac() + link.select(id) + } + @objc private func openArrangement() { arrangementWindow.show() } diff --git a/mac/Sources/DroidBridge/ArrangementWindow.swift b/mac/Sources/DroidBridge/ArrangementWindow.swift index 166a7e3..ca7ae2d 100644 --- a/mac/Sources/DroidBridge/ArrangementWindow.swift +++ b/mac/Sources/DroidBridge/ArrangementWindow.swift @@ -9,8 +9,15 @@ final class ArrangementModel: ObservableObject { @Published var displays: [DisplayInfo] = [] @Published var device: CGRect? @Published var deviceName: String? + /// The device screen as it is now; when it turns or a foldable changes screens, its rectangle follows. @Published var deviceSize: CGSize? { - didSet { if oldValue != deviceSize { reload() } } + didSet { + guard oldValue != deviceSize else { return } + reload() + if let r = device, let turned = EdgeGeometry.reshaped(device: r, aspect: aspect, displays: bounds) { + commit(turned) + } + } } /// Drag offset of the device, in global points. @Published var drag: CGSize? diff --git a/mac/Sources/DroidBridge/DeviceLink.swift b/mac/Sources/DroidBridge/DeviceLink.swift index ac33b05..2971fd6 100644 --- a/mac/Sources/DroidBridge/DeviceLink.swift +++ b/mac/Sources/DroidBridge/DeviceLink.swift @@ -10,15 +10,32 @@ final class DeviceLink { case waiting case connecting(String) case connected(model: String, width: Int, height: Int) + case chooseDevice case failed(String) } - static let serverVersion = "0.1.0" + /// An Android device, possibly reachable both by USB and over Wi-Fi. + struct Device: Equatable { + /// The hardware serial (ro.serialno), the same over USB and Wi-Fi. + let id: String + let model: String + var usbSerial: String? + var wirelessSerial: String? + } + + enum Transport { case usb, wifi } + + static let serverVersion = "0.2.0" private static let remoteJar = "/data/local/tmp/droidbridge-server.jar" private let log = Logger(subsystem: "dev.droidbridge", category: "link") var onState: ((State) -> Void)? var onMessage: ((Wire.Message) -> Void)? + /// The devices adb sees now (main queue). + private(set) var devices: [Device] = [] + /// How the current connection reaches the device. + private(set) var transport: Transport = .usb + private var hardwareSerials: [String: String] = [:] private(set) var state: State = .waiting { didSet { if state != oldValue { let s = state; DispatchQueue.main.async { self.onState?(s) } } } @@ -66,23 +83,79 @@ final class DeviceLink { Thread.sleep(forTimeInterval: 3) continue } - guard let device = firstDevice(adb) else { - state = .waiting + let found = listDevices(adb) + DispatchQueue.main.async { self.devices = found } + let chosen = Settings.deviceSerial + let target: Device? + if let chosen { + target = found.first { $0.id == chosen } + } else { + target = found.count == 1 ? found.first : nil + } + guard let device = target, let serial = device.usbSerial ?? device.wirelessSerial else { + if found.count > 1, chosen == nil || !found.contains(where: { $0.id == chosen }) { + state = chosen == nil ? .chooseDevice : .waiting + } else { + state = .waiting + } Thread.sleep(forTimeInterval: 2) continue } + transport = device.usbSerial != nil ? .usb : .wifi do { - try connect(adb: adb, serial: device) + try connect(adb: adb, serial: serial) readUntilClosed() } catch { log.error("connect failed: \(error.localizedDescription, privacy: .public)") state = .failed(error.localizedDescription) } disconnect() - Thread.sleep(forTimeInterval: 2) + Thread.sleep(forTimeInterval: 1) + } + } + + /// Picks a device from the menu; reconnects to it. + func select(_ id: String) { + Settings.deviceSerial = id + queue.async { [weak self] in self?.dropConnection() } + } + + /// Ends the current connection; the loop picks the device again. + private func dropConnection() { + writeQueue.sync { + if socket >= 0 { shutdown(socket, SHUT_RDWR) } } } + // MARK: - Devices + + private func listDevices(_ adb: String) -> [Device] { + guard let out = try? run(adb, ["devices", "-l"], timeout: 5) else { return [] } + var byID: [String: Device] = [:] + var order: [String] = [] + for e in AdbParsing.devices(out) where e.state == "device" { + let id: String + if e.wireless { + if let cached = hardwareSerials[e.serial] { + id = cached + } else if let s = try? run(adb, ["-s", e.serial, "shell", "getprop", "ro.serialno"], timeout: 5) + .trimmingCharacters(in: .whitespacesAndNewlines), !s.isEmpty { + hardwareSerials[e.serial] = s + id = s + } else { + id = e.serial + } + } else { + id = e.serial + } + var d = byID[id] ?? Device(id: id, model: e.model ?? id, usbSerial: nil, wirelessSerial: nil) + if e.wireless { d.wirelessSerial = e.serial } else { d.usbSerial = e.serial } + if byID[id] == nil { order.append(id) } + byID[id] = d + } + return order.compactMap { byID[$0] } + } + private func connect(adb: String, serial: String) throws { self.serial = serial state = .connecting(serial) @@ -161,26 +234,30 @@ final class DeviceLink { // MARK: - Helpers - private func firstDevice(_ adb: String) -> String? { - guard let out = try? run(adb, ["devices"]) else { return nil } - for line in out.split(separator: "\n").dropFirst() { - let parts = line.split(whereSeparator: { $0 == "\t" || $0 == " " }) - if parts.count >= 2, parts[1] == "device" { return String(parts[0]) } - } - return nil - } - @discardableResult - private func run(_ tool: String, _ args: [String]) throws -> String { + private func run(_ tool: String, _ args: [String], timeout: TimeInterval = 60) throws -> String { let p = Process() p.executableURL = URL(fileURLWithPath: tool) p.arguments = args let out = Pipe() p.standardOutput = out p.standardError = out + let done = DispatchSemaphore(value: 0) + p.terminationHandler = { _ in done.signal() } try p.run() - let data = out.fileHandleForReading.readDataToEndOfFile() - p.waitUntilExit() + var data = Data() + let reader = DispatchQueue.global() + let readDone = DispatchSemaphore(value: 0) + reader.async { + data = out.fileHandleForReading.readDataToEndOfFile() + readDone.signal() + } + if done.wait(timeout: .now() + timeout) == .timedOut { + p.terminate() + _ = done.wait(timeout: .now() + 1) + throw LinkError("adb \(args.joined(separator: " ")): timed out") + } + _ = readDone.wait(timeout: .now() + 2) let text = String(decoding: data, as: UTF8.self) guard p.terminationStatus == 0 else { throw LinkError("adb \(args.joined(separator: " ")): \(text)") } return text diff --git a/mac/Sources/DroidBridge/Settings.swift b/mac/Sources/DroidBridge/Settings.swift index 978e377..2f3601d 100644 --- a/mac/Sources/DroidBridge/Settings.swift +++ b/mac/Sources/DroidBridge/Settings.swift @@ -22,6 +22,12 @@ struct Settings { } } + /// The device to use (hardware serial); nil = the only one connected. + static var deviceSerial: String? { + get { d.string(forKey: "deviceSerial") } + set { d.set(newValue, forKey: "deviceSerial") } + } + static var speed: Double { get { d.object(forKey: "speed") as? Double ?? 3.0 } set { d.set(newValue, forKey: "speed") } diff --git a/mac/Sources/DroidBridgeCore/AdbParsing.swift b/mac/Sources/DroidBridgeCore/AdbParsing.swift new file mode 100644 index 0000000..e2fa345 --- /dev/null +++ b/mac/Sources/DroidBridgeCore/AdbParsing.swift @@ -0,0 +1,22 @@ +/// Parsers for adb output. +public enum AdbParsing { + public struct Entry: Equatable { + public let serial: String + public let state: String + public let model: String? + public let usb: Bool + + /// Reached over the network: "ip:port" or an mDNS name. + public var wireless: Bool { serial.contains(":") || serial.contains("._adb-tls-connect.") } + } + + /// `adb devices -l` + public static func devices(_ output: String) -> [Entry] { + output.split(separator: "\n").dropFirst().compactMap { line in + let parts = line.split(whereSeparator: { $0 == " " || $0 == "\t" }).map(String.init) + guard parts.count >= 2 else { return nil } + let model = parts.first { $0.hasPrefix("model:") }.map { String($0.dropFirst(6)).replacingOccurrences(of: "_", with: " ") } + return Entry(serial: parts[0], state: parts[1], model: model, usb: parts.contains { $0.hasPrefix("usb:") }) + } + } +} diff --git a/mac/Sources/DroidBridgeCore/EdgeGeometry.swift b/mac/Sources/DroidBridgeCore/EdgeGeometry.swift index 339b92a..2dc766c 100644 --- a/mac/Sources/DroidBridgeCore/EdgeGeometry.swift +++ b/mac/Sources/DroidBridgeCore/EdgeGeometry.swift @@ -254,6 +254,30 @@ public enum EdgeGeometry { } } + /// When the device screen changes shape (it turns, or a foldable switches between its screens), + /// the rectangle takes the new aspect, keeping its longer side and the sides that touch displays. + /// Nil when nothing changes or the new rectangle would overlap a display or touch none. + public static func reshaped(device r: CGRect, aspect: CGFloat, displays: [CGRect]) -> CGRect? { + guard aspect > 0, r.height > 0, abs(r.width / r.height - aspect) / aspect > 0.02 else { return nil } + let long = max(r.width, r.height) + let w = aspect >= 1 ? long : long * aspect + let h = aspect >= 1 ? long / aspect : long + var x = r.midX - w / 2 + var y = r.midY - h / 2 + for p in passages(device: r, displays: displays) { + switch p.edge { + case .right: x = r.minX + case .left: x = r.maxX - w + case .top: y = r.maxY - h + case .bottom: y = r.minY + } + } + let next = CGRect(x: x, y: y, width: w, height: h) + let overlaps = displays.contains { $0.intersection(next).width > 0.5 && $0.intersection(next).height > 0.5 } + guard !overlaps, !passages(device: next, displays: displays).isEmpty else { return nil } + return next + } + /// The default device rectangle: against the outermost display's free right edge, centered, /// `size` times that display's height. public static func defaultDevice(displays: [CGRect], aspect: CGFloat, size: CGFloat = 0.6) -> CGRect? { diff --git a/mac/Tests/DroidBridgeCoreTests/CoreTests.swift b/mac/Tests/DroidBridgeCoreTests/CoreTests.swift index f3ed5fb..92564f5 100644 --- a/mac/Tests/DroidBridgeCoreTests/CoreTests.swift +++ b/mac/Tests/DroidBridgeCoreTests/CoreTests.swift @@ -161,6 +161,31 @@ final class DeviceRectTests: XCTestCase { XCTAssertNil(EdgeGeometry.snap(device: CGRect(x: 5000, y: 100, width: 300, height: 300), displays: displays, reach: 24)) } + func testReorientedKeepsTheTouchingSide() { + // Portrait, right of the MacBook: turning it keeps its left side on the MacBook's right edge. + let r = CGRect(x: 3840, y: 800, width: 300, height: 600) + let t = EdgeGeometry.reshaped(device: r, aspect: 2, displays: displays) + XCTAssertEqual(t, CGRect(x: 3840, y: 950, width: 600, height: 300)) + XCTAssertNil(EdgeGeometry.reshaped(device: r, aspect: 0.5, displays: displays)) + } + + func testReorientedInTheCorner() { + // Right of the external display and above the MacBook: both sides stay. + let r = CGRect(x: 1920, y: 94, width: 530, height: 400) + let t = EdgeGeometry.reshaped(device: r, aspect: 0.75, displays: displays) + XCTAssertEqual(t, CGRect(x: 1920, y: -36, width: 397.5, height: 530)) + XCTAssertEqual(EdgeGeometry.passages(device: t!, displays: displays).count, 2) + } + + func testReshapedForANarrowerScreen() { + // A foldable switching from its inner screen (2448x1848) to the cover one (1248x1972). + let r = CGRect(x: 3840, y: 800, width: 400, height: 600) + let t = EdgeGeometry.reshaped(device: r, aspect: 1248.0 / 1972.0, displays: displays) + XCTAssertEqual(t?.minX, 3840) + XCTAssertEqual(t?.height ?? 0, 600, accuracy: 0.01) + XCTAssertEqual(t?.width ?? 0, 600 * 1248.0 / 1972.0, accuracy: 0.01) + } + func testDefaultDevice() { let r = EdgeGeometry.defaultDevice(displays: displays, aspect: 0.5) XCTAssertEqual(r?.minX, 3840) @@ -242,3 +267,24 @@ final class KeyboardTests: XCTestCase { XCTAssertEqual(k.pressed.count, 6) } } + +final class AdbParsingTests: XCTestCase { + func testDevices() { + let out = """ + List of devices attached + 1901092548006978 device usb:1179648X product:glasses model:RG_glasses device:glasses transport_id:115 + RQGL8028M4Z device usb:17825792X product:h8qxxx model:SM_F971B device:h8q transport_id:114 + 10.50.2.9:46557 device product:h8qxxx model:SM_F971B device:h8q transport_id:116 + ZY22 unauthorized usb:1-2 transport_id:3 + + """ + let d = AdbParsing.devices(out) + XCTAssertEqual(d.count, 4) + XCTAssertEqual(d[1].model, "SM F971B") + XCTAssertTrue(d[1].usb) + XCTAssertFalse(d[1].wireless) + XCTAssertTrue(d[2].wireless) + XCTAssertFalse(d[2].usb) + XCTAssertEqual(d[3].state, "unauthorized") + } +} diff --git a/mac/scripts/build-app.sh b/mac/scripts/build-app.sh index af76e0e..df1a49f 100755 --- a/mac/scripts/build-app.sh +++ b/mac/scripts/build-app.sh @@ -3,7 +3,7 @@ # Env: DROIDBRIDGE_VERSION (default 0.1.0), SIGN_IDENTITY (default "-", ad-hoc), SERVER_JAR (prebuilt jar). set -euo pipefail cd "$(dirname "$0")/.." -VERSION=${DROIDBRIDGE_VERSION:-0.1.0} +VERSION=${DROIDBRIDGE_VERSION:-0.2.0} IDENTITY=${SIGN_IDENTITY:--} APP=dist/DroidBridge.app diff --git a/protocol/README.md b/protocol/README.md index ce079fb..5902662 100644 --- a/protocol/README.md +++ b/protocol/README.md @@ -31,7 +31,7 @@ All integers are big-endian. `side` is a side of the Android screen: | Type | Name | Payload | |---|---|---| -| 0x81 | DEVICE | u16 protocol version, u16 width, u16 height, UTF-8 model; sent first | +| 0x81 | DEVICE | u16 protocol version, u16 width, u16 height, UTF-8 model; sent first, and again when the screen turns | | 0x82 | EDGE | u8 side, u16 ratio: the pointer was pushed out through a return stretch | | 0x83 | CLIPBOARD | UTF-8 text copied on the device | | 0x84 | PONG | empty | diff --git a/server/build.sh b/server/build.sh index b20c88b..b5cec6d 100755 --- a/server/build.sh +++ b/server/build.sh @@ -3,7 +3,7 @@ # build_without_gradle.sh. Needs ANDROID_HOME with platforms/android-36 and build-tools/36.0.0. set -euo pipefail -VERSION=${DROIDBRIDGE_VERSION:-0.1.0} +VERSION=${DROIDBRIDGE_VERSION:-0.2.0} PLATFORM=${ANDROID_PLATFORM:-36} BUILD_TOOLS=${ANDROID_BUILD_TOOLS:-36.0.0} : "${ANDROID_HOME:?set ANDROID_HOME}" diff --git a/server/src/dev/droidbridge/PointerProbe.java b/server/src/dev/droidbridge/PointerProbe.java index 39d9f66..317044d 100644 --- a/server/src/dev/droidbridge/PointerProbe.java +++ b/server/src/dev/droidbridge/PointerProbe.java @@ -41,7 +41,7 @@ public String toString() { } } - private static final Pattern FRAME = Pattern.compile("logicalFrame=\\[0, 0, (\\d+), (\\d+)\\]"); + private static final Pattern FRAME = Pattern.compile("orientation=(\\d).*?logicalFrame=\\[0, 0, (\\d+), (\\d+)\\]"); private final IBinder input; private int deviceId = -1; @@ -82,13 +82,31 @@ Sample sample() throws IOException { } int width = 0; int height = 0; + int rotation = 0; int cursor = dump.indexOf("MouseCursorController"); Matcher frame = FRAME.matcher(dump); if (cursor >= 0 && frame.find(cursor) || frame.find(0)) { - width = Integer.parseInt(frame.group(1)); - height = Integer.parseInt(frame.group(2)); + rotation = Integer.parseInt(frame.group(1)); + width = Integer.parseInt(frame.group(2)); + height = Integer.parseInt(frame.group(3)); + } + return toLogical(Float.parseFloat(pointer.group(1)), Float.parseFloat(pointer.group(2)), rotation, width, height); + } + + /** + * The dispatcher reports pointers in the display's unrotated (physical) coordinates; the frame is + * the rotated (logical) one. Turns a physical position into a logical one. + */ + static Sample toLogical(float px, float py, int rotation, int width, int height) { + // Physical size: the logical one, swapped when turned a quarter. + int pw = rotation % 2 == 0 ? width : height; + int ph = rotation % 2 == 0 ? height : width; + switch (rotation) { + case 1: return new Sample(py, pw - 1 - px, width, height); + case 2: return new Sample(pw - 1 - px, ph - 1 - py, width, height); + case 3: return new Sample(ph - 1 - py, px, width, height); + default: return new Sample(px, py, width, height); } - return new Sample(Float.parseFloat(pointer.group(1)), Float.parseFloat(pointer.group(2)), width, height); } private String dump() throws IOException { diff --git a/server/src/dev/droidbridge/Session.java b/server/src/dev/droidbridge/Session.java index 3b63021..51d892b 100644 --- a/server/src/dev/droidbridge/Session.java +++ b/server/src/dev/droidbridge/Session.java @@ -50,6 +50,8 @@ final class Session { private int pushed; private boolean edgeSent; private int buttons; + private float gainX; + private float gainY; private final Object clipLock = new Object(); private String lastClipFromMac; @@ -70,6 +72,7 @@ final class Session { void run() throws IOException { int[] size = displaySize(); sendDevice(size[0], size[1]); + watchRotation(size); if (clipboard != null) { // Clipboard callbacks need a looper; the main thread runs one. new Handler(Looper.getMainLooper()).post(() -> clipboard.addPrimaryClipChangedListener(this::onDeviceClipboard)); @@ -191,14 +194,15 @@ private void enter(int side, float ratio, float[][] spans) throws IOException { if (s != null && s.width > 0) { float length = horizontalEdge ? s.width - 1 : s.height - 1; float target = ratio * length; - float gain = 0; + // Pixels per count along this axis, learned on earlier entries. + float gain = horizontalEdge ? gainX : gainY; for (int attempt = 0; attempt < 4; attempt++) { float now = horizontalEdge ? s.x : s.y; float error = target - now; if (Math.abs(error) < 12) { break; } - int counts = gain > 0 ? Math.round(error / gain) : (int) Math.signum(error) * 60; + int counts = gain > 0 ? Math.round(error / gain) : (int) Math.signum(error) * 200; walk(horizontalEdge, counts); sleep(15); PointerProbe.Sample next = probe.sample(); @@ -206,8 +210,13 @@ private void enter(int side, float ratio, float[][] spans) throws IOException { break; } float moved = (horizontalEdge ? next.x : next.y) - now; - if (counts != 0 && moved * counts > 0) { + if (counts != 0 && moved * counts > 0 && Math.abs(counts) >= 20) { gain = moved / counts; + if (horizontalEdge) { + gainX = gain; + } else { + gainY = gain; + } } s = next; } @@ -220,14 +229,14 @@ private void enter(int side, float ratio, float[][] spans) throws IOException { returns = spans; } - /** Moves along one axis in small steps, so acceleration stays predictable. */ + /** Moves along one axis in steps of the same size and pace, so the acceleration stays the same. */ private void walk(boolean horizontal, int counts) throws IOException { - int step = counts > 0 ? 10 : -10; + int step = counts > 0 ? 50 : -50; while (counts != 0) { int s = Math.abs(counts) < Math.abs(step) ? counts : step; mouse.input(new byte[] {0, (byte) (horizontal ? s : 0), (byte) (horizontal ? 0 : s), 0, 0}); counts -= s; - sleep(2); + sleep(1); } } @@ -363,6 +372,43 @@ private synchronized void send(int type, byte[] payload) throws IOException { out.flush(); } + /** Tells the Mac when the screen turns (its size swaps), so its arrangement can turn too. */ + private void watchRotation(int[] initial) { + final int[] last = initial.clone(); + Handler main = new Handler(Looper.getMainLooper()); + main.post(() -> { + android.hardware.display.DisplayManager dm = (android.hardware.display.DisplayManager) com.genymobile.scrcpy.FakeContext.get() + .getSystemService(android.content.Context.DISPLAY_SERVICE); + dm.registerDisplayListener(new android.hardware.display.DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + } + + @Override + public void onDisplayRemoved(int displayId) { + } + + @Override + public void onDisplayChanged(int displayId) { + if (displayId != 0) { + return; + } + int[] size = displaySize(); + if (size[0] > 0 && (size[0] != last[0] || size[1] != last[1])) { + last[0] = size[0]; + last[1] = size[1]; + Log.i("display is now " + size[0] + "x" + size[1]); + try { + sendDevice(size[0], size[1]); + } catch (IOException e) { + Log.w("rotation: " + e.getMessage()); + } + } + } + }, main); + }); + } + private int[] displaySize() { android.util.DisplayMetrics m = new android.util.DisplayMetrics(); try { diff --git a/tools/devtest.py b/tools/devtest.py index afa91a1..bdba854 100644 --- a/tools/devtest.py +++ b/tools/devtest.py @@ -5,7 +5,7 @@ ADB = "adb" REMOTE = "/data/local/tmp/droidbridge-server.jar" -VERSION = "0.1.0" +VERSION = "0.2.0" LEFT, RIGHT, TOP, BOTTOM = 0, 1, 2, 3 def adb(*a): @@ -56,12 +56,13 @@ def main(): jar, test = sys.argv[1], sys.argv[2] S = Server(jar) try: - if test in ("enter", "edge"): - for ratio in (0.25, 0.75): - t0 = time.time(); S.enter(LEFT, ratio); S.send(0x07) - t, _ = S.recv(5); print(f"enter left {ratio}: {time.time()-t0:.2f}s (pong={t==0x84})") - time.sleep(0.5) - if test == "edge": + if test.split(":")[0] in ("enter", "edge"): + side = {"left": LEFT, "right": RIGHT, "top": TOP, "bottom": BOTTOM}[test.split(":")[1] if ":" in test else "left"] + for ratio in (0.1, 0.5, 0.9, 0.25, 0.75): + t0 = time.time(); S.enter(side, ratio); S.send(0x07) + t, _ = S.recv(5); print(f"enter side {side} {ratio}: {time.time()-t0:.2f}s (pong={t==0x84})") + time.sleep(0.3) + if test.startswith("edge"): for _ in range(40): S.mouse(8, 0); time.sleep(0.008) # into the screen t, p = S.recv(0.3); print("after moving in, unexpected:", t) t0 = time.time(); got = None