#include <drogon/HttpResponse.h>
#include <drogon/HttpAppFramework.h>
#include <drogon/WebSocketController.h>
#include <devices.h>
#include <sqlite3.h>
#include <json/json.h>
#include <fstream>
#include <sstream>
#include <systemd/sd-bus.h>
#include <systemd/sd-device.h>
#include <systemd/sd-event.h>
#include <filesystem>
#include <unordered_map>
#include <set>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <atomic>
#include <chrono>
#include <algorithm>
#include <cctype>
#include <random>
#include <cstdio>
#include <sys/stat.h>
#include "utils.h"
#include "include/audit.h"

using namespace drogon;

// Forward declare snapshot helper accessible to other scopes in this TU
namespace provisioner { std::string getTopologySnapshotString(); }

// WebSocket controller for device topology streaming
class DevicesWebSocketController : public drogon::WebSocketController<DevicesWebSocketController> {
public:
    static std::vector<drogon::WebSocketConnectionPtr> subscribers;
    static std::mutex subscribersMutex;

    static void broadcast(const std::string &message) {
        std::lock_guard<std::mutex> lock(subscribersMutex);
        auto it = subscribers.begin();
        while (it != subscribers.end()) {
            if ((*it)->connected()) {
                (*it)->send(message);
                ++it;
            } else {
                it = subscribers.erase(it);
            }
        }
    }

    void handleNewMessage(const drogon::WebSocketConnectionPtr& wsConnPtr, std::string&& message, const drogon::WebSocketMessageType& type) override {
        // No-op for now; could support subscription filters later
        (void)wsConnPtr; (void)message; (void)type;
    }

    void handleNewConnection(const drogon::HttpRequestPtr& req, const drogon::WebSocketConnectionPtr& wsConnPtr) override {
        (void)req;
        {
            std::lock_guard<std::mutex> lock(subscribersMutex);
            subscribers.push_back(wsConnPtr);
        }
        // Send initial snapshot
        wsConnPtr->send(provisioner::getTopologySnapshotString());
    }

    void handleConnectionClosed(const drogon::WebSocketConnectionPtr& wsConnPtr) override {
        std::lock_guard<std::mutex> lock(subscribersMutex);
        subscribers.erase(std::remove(subscribers.begin(), subscribers.end(), wsConnPtr), subscribers.end());
    }

    WS_PATH_LIST_BEGIN
    WS_PATH_ADD("/ws/devices");
    WS_PATH_LIST_END
};

std::vector<drogon::WebSocketConnectionPtr> DevicesWebSocketController::subscribers;
std::mutex DevicesWebSocketController::subscribersMutex;

// (removed incorrect forward declaration of anonymous-namespace function)

namespace provisioner {

    struct Device {
        std::string serial;
        std::string port;
        std::string ip_address;
        std::string state;
        std::string image;
    };

    struct DeviceList {
        std::vector<Device> devices;
    };

    // Representation of a USB node (hub or device)
    struct UsbNode {
        std::string id;          // unique path id (e.g. 1-1.4)
        std::string parentId;    // parent path id or empty for root
        std::string vendor;      // idVendor
        std::string product;     // idProduct
        std::string productName; // textual product name (e.g., BCM2712 Boot)
        std::string serial;      // serial (if available)
        bool isHub{false};
        // Optional provisioning info
        std::string state;
        std::string image;
        std::string ip;
        std::string model;       // device model/type (e.g., CM5, 4B, Zero 2 W)
        int portCount{0};        // number of ports if hub (from maxchild)
        bool isPlaceholder{false};
        int speed{0};            // USB speed in Mbps (e.g., 480 for USB2, 5000 for USB3)
        // Two-digit hex of bits 4..11 of the OTP USER_BOARDREV (e.g. "17"
        // for Pi 5, "18" for CM5).  Captured by the bootstrap script via
        // `rpiboot -j` and persisted in state.db as generic device metadata.
        std::string boardType;
    };

    // Tracker state
    namespace {
        std::thread topologyThread;
        std::atomic<bool> topologyRunning{false};
        std::mutex topologyMutex;
        // Map id -> node
        std::unordered_map<std::string, UsbNode> currentTopology;

        // Event-driven scan trigger: USB uevents (sd_device_monitor) and
        // POSTs to /internal/state-changed from the provisioning scripts push
        // scanRequested=true and notify scanCv, collapsing the worker's wait.
        // A 10s wait_for backstop covers anything that slips past both event
        // sources.
        std::mutex scanMu;
        std::condition_variable scanCv;
        bool scanRequested{false};
        std::thread eventSourceThread;
        sd_event *eventSourceLoop{nullptr};

        void requestScan() {
            {
                std::lock_guard<std::mutex> lk(scanMu);
                scanRequested = true;
            }
            scanCv.notify_all();
        }
        // App start time in milliseconds since epoch; used to ignore stale DB rows
        std::atomic<long long> appStartMs{0};
        
        // Test mode state for fake device injection
        std::atomic<bool> testModeEnabled{false};
        std::unordered_map<std::string, UsbNode> testTopology;
        
        // USB spec constants
        constexpr int USB_MAX_DEVICES = 127;      // Maximum devices per host controller
        constexpr int USB_MAX_HUB_DEPTH = 5;      // Maximum external hubs in chain (7 tiers total)
        
        // Generate a fake serial number
        std::string generateFakeSerial(int index) {
            char buf[32];
            snprintf(buf, sizeof(buf), "TEST%08x", 0x10000000 + index);
            return buf;
        }
        
        // Generate a fake IP address
        std::string generateFakeIP(int index) {
            return "192.168.100." + std::to_string((index % 254) + 1);
        }
        
        // Test scenario generators
        std::unordered_map<std::string, UsbNode> generateDirectDevices(int count) {
            // Devices connected directly to root ports (no hubs)
            std::unordered_map<std::string, UsbNode> nodes;
            
            // Server root
            UsbNode server;
            server.id = "server";
            server.parentId = "";
            server.isHub = true;
            server.vendor = "Provisioner";
            server.product = "Server";
            nodes["server"] = server;
            
            // Add root hub placeholder for USB 3 (bus 3)
            UsbNode rootHub;
            rootHub.id = "3-0";
            rootHub.parentId = "server";
            rootHub.isHub = true;
            rootHub.portCount = std::min(count, 4);
            nodes["3-0"] = rootHub;
            
            // Direct devices on different root ports
            const char* states[] = {"bootstrap", "triage", "provisioning", "complete", "error"};
            const char* models[] = {"CM5", "CM4", "Pi 5", "Pi 4B", "Zero 2 W"};
            const int modelGens[] = {5, 4, 5, 4, 0};
            
            for (int i = 0; i < count && i < 4; ++i) {
                UsbNode dev;
                dev.id = "3-" + std::to_string(i + 1);
                dev.parentId = "server";
                dev.isHub = false;
                dev.vendor = "2e8a";
                dev.product = "000" + std::to_string(i);
                dev.productName = "BCM2712 Boot";
                dev.serial = generateFakeSerial(i);
                dev.state = states[i % 5];
                dev.ip = generateFakeIP(i);
                dev.model = models[i % 5];
                nodes[dev.id] = dev;
            }
            
            return nodes;
        }
        
        std::unordered_map<std::string, UsbNode> generateHubWithDevices(int hubPorts, int deviceCount) {
            // Single hub with multiple devices
            std::unordered_map<std::string, UsbNode> nodes;
            
            // Server root
            UsbNode server;
            server.id = "server";
            server.parentId = "";
            server.isHub = true;
            server.vendor = "Provisioner";
            server.product = "Server";
            nodes["server"] = server;
            
            // Root hub
            UsbNode rootHub;
            rootHub.id = "3-0";
            rootHub.parentId = "server";
            rootHub.isHub = true;
            rootHub.portCount = 2;
            nodes["3-0"] = rootHub;
            
            // USB Hub on port 1
            UsbNode hub;
            hub.id = "3-1";
            hub.parentId = "server";
            hub.isHub = true;
            hub.vendor = "0bda";
            hub.product = "5411";
            hub.productName = "USB3.0 Hub";
            hub.portCount = hubPorts;
            nodes["3-1"] = hub;
            
            // Devices under the hub
            const char* states[] = {"bootstrap", "triage", "provisioning", "complete", "error"};
            const char* models[] = {"CM5", "CM4", "Pi 5", "Pi 4B", "Zero 2 W"};
            
            for (int i = 0; i < deviceCount && i < hubPorts; ++i) {
                UsbNode dev;
                dev.id = "3-1." + std::to_string(i + 1);
                dev.parentId = "3-1";
                dev.isHub = false;
                dev.vendor = "2e8a";
                dev.product = "000" + std::to_string(i);
                dev.productName = "BCM2712 Boot";
                dev.serial = generateFakeSerial(100 + i);
                dev.state = states[i % 5];
                dev.ip = generateFakeIP(100 + i);
                dev.model = models[i % 5];
                nodes[dev.id] = dev;
            }
            
            // Add placeholders for empty ports
            for (int i = deviceCount; i < hubPorts; ++i) {
                UsbNode ph;
                ph.id = "3-1." + std::to_string(i + 1);
                ph.parentId = "3-1";
                ph.isHub = false;
                ph.isPlaceholder = true;
                nodes[ph.id] = ph;
            }
            
            return nodes;
        }
        
        std::unordered_map<std::string, UsbNode> generateNestedHubs(int depth, int devicesPerHub) {
            // Nested hubs up to specified depth (max 5)
            std::unordered_map<std::string, UsbNode> nodes;
            depth = std::min(depth, USB_MAX_HUB_DEPTH);
            
            // Server root
            UsbNode server;
            server.id = "server";
            server.parentId = "";
            server.isHub = true;
            server.vendor = "Provisioner";
            server.product = "Server";
            nodes["server"] = server;
            
            // Root hub
            UsbNode rootHub;
            rootHub.id = "3-0";
            rootHub.parentId = "server";
            rootHub.isHub = true;
            rootHub.portCount = 2;
            nodes["3-0"] = rootHub;
            
            // Build chain of hubs
            std::string parentId = "server";
            std::string currentId = "3-1";
            const char* states[] = {"bootstrap", "triage", "provisioning", "complete"};
            const char* models[] = {"CM5", "CM4", "Pi 5", "Pi 4B"};
            int deviceIndex = 0;
            
            for (int d = 0; d < depth; ++d) {
                // Add hub at this depth
                UsbNode hub;
                hub.id = currentId;
                hub.parentId = parentId;
                hub.isHub = true;
                hub.vendor = "0bda";
                hub.product = "5411";
                hub.productName = "USB3.0 Hub (Tier " + std::to_string(d + 1) + ")";
                hub.portCount = devicesPerHub + 1; // +1 for next hub
                nodes[hub.id] = hub;
                
                // Add devices at this level
                for (int i = 0; i < devicesPerHub; ++i) {
                    UsbNode dev;
                    dev.id = currentId + "." + std::to_string(i + 1);
                    dev.parentId = currentId;
                    dev.isHub = false;
                    dev.vendor = "2e8a";
                    dev.product = "0003";
                    dev.productName = "BCM2712 Boot";
                    dev.serial = generateFakeSerial(200 + deviceIndex);
                    dev.state = states[deviceIndex % 4];
                    dev.ip = generateFakeIP(200 + deviceIndex);
                    dev.model = models[deviceIndex % 4];
                    nodes[dev.id] = dev;
                    deviceIndex++;
                }
                
                // Prepare for next level
                parentId = currentId;
                currentId = currentId + "." + std::to_string(devicesPerHub + 1);
            }
            
            return nodes;
        }
        
        std::unordered_map<std::string, UsbNode> generateMaxTopology() {
            // Maximum USB topology: 127 devices with hub hierarchy
            std::unordered_map<std::string, UsbNode> nodes;
            
            // Server root
            UsbNode server;
            server.id = "server";
            server.parentId = "";
            server.isHub = true;
            server.vendor = "Provisioner";
            server.product = "Server";
            nodes["server"] = server;
            
            // Root hubs for USB2 (bus 1) and USB3 (bus 3)
            for (int bus : {1, 3}) {
                UsbNode rootHub;
                rootHub.id = std::to_string(bus) + "-0";
                rootHub.parentId = "server";
                rootHub.isHub = true;
                rootHub.portCount = 4;
                nodes[rootHub.id] = rootHub;
            }
            
            const char* states[] = {"bootstrap", "triage", "provisioning", "complete", "error"};
            const char* models[] = {"CM5", "CM4", "Pi 5", "Pi 4B", "Zero 2 W"};
            int deviceIndex = 0;
            int totalDevices = 0;
            
            // Create hierarchical structure to reach ~127 devices
            // 4 root ports × 7-port hubs × 4 devices per hub = 112 devices + hubs
            for (int bus : {1, 3}) {
                for (int rootPort = 1; rootPort <= 2 && totalDevices < USB_MAX_DEVICES - 10; ++rootPort) {
                    std::string hubId = std::to_string(bus) + "-" + std::to_string(rootPort);
                    
                    // Add 7-port hub
                    UsbNode hub;
                    hub.id = hubId;
                    hub.parentId = "server";
                    hub.isHub = true;
                    hub.vendor = "0bda";
                    hub.product = "5411";
                    hub.productName = "USB3.0 Hub";
                    hub.portCount = 7;
                    nodes[hub.id] = hub;
                    totalDevices++;
                    
                    // Add devices to hub
                    for (int port = 1; port <= 7 && totalDevices < USB_MAX_DEVICES; ++port) {
                        // Every 3rd port gets a sub-hub with more devices
                        if (port % 3 == 0 && totalDevices < USB_MAX_DEVICES - 5) {
                            std::string subHubId = hubId + "." + std::to_string(port);
                            UsbNode subHub;
                            subHub.id = subHubId;
                            subHub.parentId = hubId;
                            subHub.isHub = true;
                            subHub.vendor = "0bda";
                            subHub.product = "5411";
                            subHub.productName = "USB3.0 Sub-Hub";
                            subHub.portCount = 4;
                            nodes[subHub.id] = subHub;
                            totalDevices++;
                            
                            // Devices on sub-hub
                            for (int subPort = 1; subPort <= 4 && totalDevices < USB_MAX_DEVICES; ++subPort) {
                                UsbNode dev;
                                dev.id = subHubId + "." + std::to_string(subPort);
                                dev.parentId = subHubId;
                                dev.isHub = false;
                                dev.vendor = "2e8a";
                                dev.product = "0003";
                                dev.productName = "BCM2712 Boot";
                                dev.serial = generateFakeSerial(deviceIndex);
                                dev.state = states[deviceIndex % 5];
                                dev.ip = generateFakeIP(deviceIndex);
                                dev.model = models[deviceIndex % 5];
                                nodes[dev.id] = dev;
                                deviceIndex++;
                                totalDevices++;
                            }
                        } else {
                            // Regular device
                            UsbNode dev;
                            dev.id = hubId + "." + std::to_string(port);
                            dev.parentId = hubId;
                            dev.isHub = false;
                            dev.vendor = "2e8a";
                            dev.product = "0003";
                            dev.productName = "BCM2712 Boot";
                            dev.serial = generateFakeSerial(deviceIndex);
                            dev.state = states[deviceIndex % 5];
                            dev.ip = generateFakeIP(deviceIndex);
                            dev.model = models[deviceIndex % 5];
                            nodes[dev.id] = dev;
                            deviceIndex++;
                            totalDevices++;
                        }
                    }
                }
            }
            
            LOG_INFO << "Generated max topology with " << totalDevices << " devices";
            return nodes;
        }
        
        std::unordered_map<std::string, UsbNode> generateMixedTopology() {
            // Mixed topology: direct devices, hubs, and nested hubs
            std::unordered_map<std::string, UsbNode> nodes;
            
            // Server root
            UsbNode server;
            server.id = "server";
            server.parentId = "";
            server.isHub = true;
            server.vendor = "Provisioner";
            server.product = "Server";
            nodes["server"] = server;
            
            // Root hubs
            for (int bus : {1, 3}) {
                UsbNode rootHub;
                rootHub.id = std::to_string(bus) + "-0";
                rootHub.parentId = "server";
                rootHub.isHub = true;
                rootHub.portCount = 4;
                nodes[rootHub.id] = rootHub;
            }
            
            const char* states[] = {"bootstrap", "triage", "provisioning", "complete", "error"};
            const char* models[] = {"CM5", "CM4", "Pi 5", "Pi 4B", "Zero 2 W"};
            int deviceIndex = 0;
            
            // USB3 bus: Direct device on port 1
            {
                UsbNode dev;
                dev.id = "3-1";
                dev.parentId = "server";
                dev.isHub = false;
                dev.vendor = "2e8a";
                dev.product = "0003";
                dev.productName = "BCM2712 Boot";
                dev.serial = generateFakeSerial(deviceIndex++);
                dev.state = "complete";
                dev.ip = generateFakeIP(0);
                dev.model = "CM5";
                nodes[dev.id] = dev;
            }
            
            // USB3 bus: Hub on port 2 with devices
            {
                UsbNode hub;
                hub.id = "3-2";
                hub.parentId = "server";
                hub.isHub = true;
                hub.vendor = "0bda";
                hub.product = "5411";
                hub.productName = "USB3.0 Hub";
                hub.portCount = 4;
                nodes[hub.id] = hub;
                
                for (int i = 1; i <= 3; ++i) {
                    UsbNode dev;
                    dev.id = "3-2." + std::to_string(i);
                    dev.parentId = "3-2";
                    dev.isHub = false;
                    dev.vendor = "2e8a";
                    dev.product = "0003";
                    dev.productName = "BCM2712 Boot";
                    dev.serial = generateFakeSerial(deviceIndex);
                    dev.state = states[deviceIndex % 5];
                    dev.ip = generateFakeIP(deviceIndex);
                    dev.model = models[deviceIndex % 5];
                    nodes[dev.id] = dev;
                    deviceIndex++;
                }
                
                // Placeholder for empty port
                UsbNode ph;
                ph.id = "3-2.4";
                ph.parentId = "3-2";
                ph.isPlaceholder = true;
                nodes[ph.id] = ph;
            }
            
            // USB2 bus: Nested hubs
            {
                UsbNode hub1;
                hub1.id = "1-1";
                hub1.parentId = "server";
                hub1.isHub = true;
                hub1.vendor = "0bda";
                hub1.product = "5411";
                hub1.productName = "USB2.0 Hub";
                hub1.portCount = 4;
                nodes[hub1.id] = hub1;
                
                // Device on hub1 port 1
                UsbNode dev1;
                dev1.id = "1-1.1";
                dev1.parentId = "1-1";
                dev1.isHub = false;
                dev1.vendor = "2e8a";
                dev1.product = "0003";
                dev1.productName = "BCM2711 Boot";
                dev1.serial = generateFakeSerial(deviceIndex++);
                dev1.state = "provisioning";
                dev1.ip = generateFakeIP(50);
                dev1.model = "CM4";
                nodes[dev1.id] = dev1;
                
                // Nested hub on hub1 port 2
                UsbNode hub2;
                hub2.id = "1-1.2";
                hub2.parentId = "1-1";
                hub2.isHub = true;
                hub2.vendor = "0bda";
                hub2.product = "5411";
                hub2.productName = "USB2.0 Sub-Hub";
                hub2.portCount = 4;
                nodes[hub2.id] = hub2;
                
                // Devices on nested hub
                for (int i = 1; i <= 2; ++i) {
                    UsbNode dev;
                    dev.id = "1-1.2." + std::to_string(i);
                    dev.parentId = "1-1.2";
                    dev.isHub = false;
                    dev.vendor = "2e8a";
                    dev.product = "0003";
                    dev.productName = "BCM2711 Boot";
                    dev.serial = generateFakeSerial(deviceIndex);
                    dev.state = states[deviceIndex % 5];
                    dev.ip = generateFakeIP(60 + i);
                    dev.model = "Pi 4B";
                    nodes[dev.id] = dev;
                    deviceIndex++;
                }
            }
            
            return nodes;
        }

        std::string readFileTrimmed(const std::filesystem::path &p) {
            try {
                std::ifstream f(p);
                if (!f.is_open()) return "";
                std::string s;
                std::getline(f, s);
                while (!s.empty() && (s.back()=='\n' || s.back()=='\r' || s.back()=='\t' || s.back()==' ')) s.pop_back();
                return s;
            } catch (...) { return ""; }
        }

        std::string baseDeviceDirName(const std::string &name) {
            // Strip interface suffix after ':' (e.g. 1-1.2:1.0 -> 1-1.2)
            auto pos = name.find(':');
            if (pos != std::string::npos) return name.substr(0, pos);
            return name;
        }

        std::string computeParentId(const std::string &id) {
            // Parent is prefix before last '.' if present, otherwise attach to server
            auto pos = id.rfind('.');
            if (pos != std::string::npos) {
                return id.substr(0, pos);
            }
            // If id contains '-' treat as directly connected to server
            return "server";
        }

        struct CompanionBusPair {
            int hsBus;
            int ssBus;
            int ssPortCount;
        };

        // Resolve companion HS/SS bus pairs by checking which root hubs
        // share a physical USB controller in sysfs.  This is authoritative
        // (no heuristic) — two root hubs are companions iff they share
        // the same parent device (the xHCI controller instance).
        std::vector<CompanionBusPair> detectCompanionBuses() {
            std::vector<CompanionBusPair> pairs;
            const std::filesystem::path usbPath("/sys/bus/usb/devices");
            if (!std::filesystem::exists(usbPath)) return pairs;

            struct BusInfo { int bus; int speed; int portCount; };
            std::unordered_map<std::string, std::vector<BusInfo>> controllerBuses;

            for (const auto &entry : std::filesystem::directory_iterator(usbPath)) {
                const std::string name = entry.path().filename().string();
                if (name.size() < 4 || name.substr(0, 3) != "usb") continue;
                int bus;
                try { bus = std::stoi(name.substr(3)); } catch (...) { continue; }

                std::error_code ec;
                auto realPath = std::filesystem::canonical(entry.path(), ec);
                if (ec) continue;
                std::string controllerPath = realPath.parent_path().string();

                int speed = 0;
                const std::string speedStr = readFileTrimmed(entry.path() / "speed");
                if (!speedStr.empty()) {
                    try { speed = std::stoi(speedStr); } catch (...) {}
                }

                int portCount = 0;
                const std::string mcStr = readFileTrimmed(entry.path() / "maxchild");
                if (!mcStr.empty()) {
                    try { portCount = std::stoi(mcStr); } catch (...) {}
                }

                controllerBuses[controllerPath].push_back({bus, speed, portCount});
            }

            for (const auto &[ctrl, buses] : controllerBuses) {
                std::vector<BusInfo> hsList, ssList;
                for (const auto &b : buses) {
                    if (b.speed >= 5000) ssList.push_back(b);
                    else hsList.push_back(b);
                }
                if (ssList.empty() || hsList.empty()) continue;
                // Sort by bus number for deterministic sequential pairing
                std::sort(hsList.begin(), hsList.end(), [](const BusInfo &a, const BusInfo &b) { return a.bus < b.bus; });
                std::sort(ssList.begin(), ssList.end(), [](const BusInfo &a, const BusInfo &b) { return a.bus < b.bus; });
                size_t count = std::min(hsList.size(), ssList.size());
                for (size_t i = 0; i < count; ++i) {
                    pairs.push_back({hsList[i].bus, ssList[i].bus, ssList[i].portCount});
                }
            }

            return pairs;
        }

        std::unordered_map<std::string, UsbNode> scanUsbSysfs() {
            std::unordered_map<std::string, UsbNode> nodes;
            // Always include the server root node
            UsbNode server;
            server.id = "server";
            server.parentId = "";
            server.isHub = true;
            server.vendor = "Provisioner";
            server.product = "Server";
            nodes.insert({server.id, server});

            const std::filesystem::path usbPath("/sys/bus/usb/devices");
            if (!std::filesystem::exists(usbPath)) {
                return nodes;
            }

            for (const auto &entry : std::filesystem::directory_iterator(usbPath)) {
                if (!entry.is_directory()) continue;
                const std::string name = entry.path().filename().string();
                if (name.find('-') == std::string::npos) continue; // skip non-device entries like usb1
                const bool isInterface = (name.find(':') != std::string::npos);
                const std::string id = baseDeviceDirName(name);

                // Create or get existing node by base id
                UsbNode &node = nodes[id];
                if (node.id.empty()) {
                    node.id = id;
                    node.parentId = computeParentId(id);
                }

                const std::filesystem::path baseDir = usbPath / id;

                // Populate from device directory when available
                if (!isInterface) {
                    const std::string v = readFileTrimmed(entry.path()/"idVendor");
                    const std::string p = readFileTrimmed(entry.path()/"idProduct");
                    if (!v.empty()) node.vendor = v;
                    if (!p.empty()) node.product = p;
                    const std::string pn = readFileTrimmed(entry.path()/"product");
                    if (!pn.empty()) node.productName = pn;
                    // Default model hint from product name; refined later by manufacturing data
                    if (node.model.empty() && !pn.empty()) node.model = pn;
                    const std::string s = readFileTrimmed(entry.path()/"serial");
                    if (!s.empty()) node.serial = s;
                    // Read USB speed (in Mbps, e.g., 480, 5000, 10000, 20000)
                    const std::string speedStr = readFileTrimmed(entry.path()/"speed");
                    if (!speedStr.empty()) {
                        try { node.speed = std::stoi(speedStr); } catch (...) {}
                    }
                }

                // Detect hubs via either device or interface class files
                const std::string bDeviceClass = std::filesystem::exists(entry.path()/"bDeviceClass")
                    ? readFileTrimmed(entry.path()/"bDeviceClass")
                    : (std::filesystem::exists(baseDir/"bDeviceClass") ? readFileTrimmed(baseDir/"bDeviceClass") : "");
                const std::string bInterfaceClass = std::filesystem::exists(entry.path()/"bInterfaceClass")
                    ? readFileTrimmed(entry.path()/"bInterfaceClass")
                    : (std::filesystem::exists(baseDir/"bInterfaceClass") ? readFileTrimmed(baseDir/"bInterfaceClass") : "");
                if (bDeviceClass == "09" || bDeviceClass == "9" || bDeviceClass == "0x09" ||
                    bInterfaceClass == "09" || bInterfaceClass == "9" || bInterfaceClass == "0x09") {
                    node.isHub = true;
                }

                // Read maxchild from whichever location provides it
                for (const auto &dir : {entry.path(), baseDir}) {
                    std::error_code ec;
                    const auto path = dir/"maxchild";
                    if (std::filesystem::exists(path, ec)) {
                        const std::string mc = readFileTrimmed(path);
                        if (!mc.empty()) {
                            try { node.portCount = std::max(node.portCount, std::stoi(mc)); } catch (...) {}
                        }
                    }
                }

                // On this platform, root hubs expose child ports as usbX-portN directories under the interface dir
                if (isInterface && id.size() >= 2 && id.rfind("-0") == id.size() - 2) {
                    for (const auto &child : std::filesystem::directory_iterator(entry.path())) {
                        const std::string childName = child.path().filename().string();
                        // Match pattern: usb<bus>-port<idx>
                        auto dashPos = childName.find("-port");
                        if (dashPos == std::string::npos) continue;
                        // Extract trailing number after "-port"
                        const std::string portStr = childName.substr(dashPos + 5);
                        try {
                            int portNum = std::stoi(portStr);
                            if (portNum > node.portCount) node.portCount = portNum;
                        } catch (...) {
                            // ignore non-numeric suffixes
                        }
                    }
                }
            }

            // Add placeholder entries for unconnected hub ports
            std::vector<UsbNode> placeholders;
            for (const auto &p : nodes) {
                const UsbNode &hub = p.second;
                if (!hub.isHub || hub.portCount <= 0) continue;
                const bool isRootHub = (hub.id.size() >= 2 && hub.id.rfind("-0") == hub.id.size() - 2);
                for (int i = 1; i <= hub.portCount; ++i) {
                    std::string childId;
                    std::string parentId;
                    if (isRootHub) {
                        // Root hub children use pattern "<bus>-<port>", and attach directly to server
                        const std::string busPrefix = hub.id.substr(0, hub.id.size() - 2);
                        childId = busPrefix + "-" + std::to_string(i);
                        parentId = "server";
                    } else {
                        childId = hub.id + "." + std::to_string(i);
                        parentId = hub.id;
                    }
                    if (nodes.find(childId) != nodes.end()) continue;
                    UsbNode ph;
                    ph.id = childId;
                    ph.parentId = parentId;
                    ph.isHub = false;
                    // Leave vendor/product empty; UI will render as placeholder by id
                    ph.serial = "";
                    ph.state = "";
                    ph.image = "";
                    ph.ip = "";
                    ph.isPlaceholder = true;
                    placeholders.push_back(std::move(ph));
                }
            }
            for (const auto &ph : placeholders) {
                nodes.insert_or_assign(ph.id, ph);
            }

            return nodes;
        }

        void enrichWithProvisioningState(std::unordered_map<std::string, UsbNode> &nodes) {
            // Build latest record per endpoint (descending ts ensures first seen is newest)
            struct DbRecord { std::string serial, state, image, ip, boardType; };
            std::unordered_map<std::string, DbRecord> latestByEndpoint;
            struct MfgRecord { std::string boardname, processor, osImageFilename; };
            std::unordered_map<std::string, MfgRecord> latestMfgBySerial;

            sqlite3* db;
            int rc = sqlite3_open("/srv/rpi-sb-provisioner/state.db", &db);
            if (rc) {
                return;
            }
            sqlite3_busy_timeout(db, 5000);
            // id (monotonic rowid) breaks ts ties: CURRENT_TIMESTAMP is
            // second-resolution, so the rapid triage->provisioner handoff
            // writes several rows in one second. Ordering on ts alone leaves
            // their order to the query planner, which can mis-pick the latest
            // state per endpoint. id reflects true insertion order.
            const char* sql = "SELECT serial, endpoint, state, image, ip_address, board_type FROM devices WHERE ts >= ? ORDER BY ts DESC, id DESC";
            sqlite3_stmt* stmt;
            rc = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
            if (rc != SQLITE_OK) {
                sqlite3_close(db);
                return;
            }
            // Bind application start time (milliseconds)
            sqlite3_bind_int64(stmt, 1, static_cast<sqlite3_int64>(appStartMs.load()));
            while (sqlite3_step(stmt) == SQLITE_ROW) {
                const unsigned char* serial = sqlite3_column_text(stmt, 0);
                const unsigned char* endpoint = sqlite3_column_text(stmt, 1);
                const unsigned char* state = sqlite3_column_text(stmt, 2);
                const unsigned char* image = sqlite3_column_text(stmt, 3);
                const unsigned char* ip = sqlite3_column_text(stmt, 4);
                const unsigned char* boardType = sqlite3_column_text(stmt, 5);
                if (!endpoint) continue;
                std::string endpointStr = reinterpret_cast<const char*>(endpoint);
                std::string boardTypeStr = boardType ? reinterpret_cast<const char*>(boardType) : std::string{};
                std::string serialStr = serial ? reinterpret_cast<const char*>(serial) : std::string{};
                if (latestByEndpoint.find(endpointStr) == latestByEndpoint.end()) {
                    latestByEndpoint.emplace(endpointStr, DbRecord{
                        serialStr,
                        state ? reinterpret_cast<const char*>(state) : std::string{},
                        image ? reinterpret_cast<const char*>(image) : std::string{},
                        ip ? reinterpret_cast<const char*>(ip) : std::string{},
                        boardTypeStr
                    });
                }
            }
            sqlite3_finalize(stmt);
            sqlite3_close(db);

            // Read manufacturing database if present for device-type inference
            {
                sqlite3* mdb = nullptr;
                int rc2 = sqlite3_open("/srv/rpi-sb-provisioner/manufacturing.db", &mdb);
                if (rc2 == SQLITE_OK) {
                    sqlite3_busy_timeout(mdb, 5000);
                    const char* msql = "SELECT serial, boardname, processor, os_image_filename FROM devices ORDER BY provision_ts DESC";
                    sqlite3_stmt* mstmt = nullptr;
                    rc2 = sqlite3_prepare_v2(mdb, msql, -1, &mstmt, nullptr);
                    if (rc2 == SQLITE_OK) {
                        while (sqlite3_step(mstmt) == SQLITE_ROW) {
                            const unsigned char* serial = sqlite3_column_text(mstmt, 0);
                            const unsigned char* boardname = sqlite3_column_text(mstmt, 1);
                            const unsigned char* processor = sqlite3_column_text(mstmt, 2);
                            const unsigned char* osImage = sqlite3_column_text(mstmt, 3);
                            if (!serial) continue;
                            std::string s = reinterpret_cast<const char*>(serial);
                            if (latestMfgBySerial.find(s) == latestMfgBySerial.end()) {
                                latestMfgBySerial.emplace(s, MfgRecord{
                                    boardname ? reinterpret_cast<const char*>(boardname) : std::string{},
                                    processor ? reinterpret_cast<const char*>(processor) : std::string{},
                                    osImage ? reinterpret_cast<const char*>(osImage) : std::string{}
                                });
                            }
                        }
                    }
                    if (mstmt) sqlite3_finalize(mstmt);
                    sqlite3_close(mdb);
                }
            }

            // Apply only when: (1) port is connected (non-placeholder), (2) we have a latest record for this endpoint
            for (const auto &p : latestByEndpoint) {
                const std::string &endpoint = p.first;
                const DbRecord &rec = p.second;
                auto it = nodes.find(endpoint);
                if (it == nodes.end()) continue; // endpoint not present in current topology
                UsbNode &n = it->second;
                if (n.isHub) continue; // never apply provisioning state to hubs
                if (n.isPlaceholder) continue; // empty port: runtime fields describe a device that isn't here
                n.state = rec.state;
                // Do not clobber existing non-empty image (used for model inference) with empty DB values
                if (!rec.image.empty()) {
                    n.image = rec.image;
                }
                n.ip = rec.ip;
                if (!rec.boardType.empty()) {
                    n.boardType = rec.boardType;
                }
                // If we also have a manufacturing record for this device by serial, use it to annotate image/model
                if (!n.serial.empty()) {
                    auto mit = latestMfgBySerial.find(n.serial);
                    if (mit != latestMfgBySerial.end()) {
                        const auto &mr = mit->second;
                        if (!mr.boardname.empty()) n.model = mr.boardname;
                        else if (!mr.processor.empty() && n.model.empty()) n.model = mr.processor;
                        if (n.image.empty() && !mr.osImageFilename.empty()) n.image = mr.osImageFilename;
                    }
                }
            }

            // Fallback: Match by serial number when endpoint lookup failed
            // This handles cases where the endpoint field in the database doesn't match the USB topology
            // (e.g., due to incorrect USB path extraction during bootstrap phase)
            std::unordered_map<std::string, DbRecord> latestBySerial;
            for (const auto &p : latestByEndpoint) {
                if (!p.second.serial.empty()) {
                    latestBySerial[p.second.serial] = p.second;
                }
            }

            for (auto &p : nodes) {
                UsbNode &n = p.second;
                // Skip if already has state (endpoint match succeeded), is placeholder, or is hub
                if (!n.state.empty() || n.isPlaceholder || n.isHub) continue;
                // Skip if no serial available
                if (n.serial.empty()) continue;
                
                auto sit = latestBySerial.find(n.serial);
                if (sit != latestBySerial.end()) {
                    const auto &rec = sit->second;
                    n.state = rec.state;
                    if (!rec.image.empty()) {
                        n.image = rec.image;
                    }
                    n.ip = rec.ip;
                    if (!rec.boardType.empty()) {
                        n.boardType = rec.boardType;
                    }
                    auto mit = latestMfgBySerial.find(n.serial);
                    if (mit != latestMfgBySerial.end()) {
                        const auto &mr = mit->second;
                        if (!mr.boardname.empty()) n.model = mr.boardname;
                        else if (!mr.processor.empty() && n.model.empty()) n.model = mr.processor;
                        if (n.image.empty() && !mr.osImageFilename.empty()) n.image = mr.osImageFilename;
                    }
                }
            }
        }

        int inferModelGeneration(const UsbNode &n) {
            // Prefer explicit image tag if available
            if (!n.model.empty()) {
                std::string img = n.model;
                std::transform(img.begin(), img.end(), img.begin(), [](unsigned char c){ return static_cast<char>(std::tolower(c)); });
                if (img.find("2712") != std::string::npos ||
                    img.find("rpi5") != std::string::npos ||
                    img.find("pi5") != std::string::npos ||
                    img.find("cm5") != std::string::npos ||
                    img.find("compute module 5") != std::string::npos)
                {
                    return 5;
                }
                if (img.find("2711") != std::string::npos ||
                    img.find("rpi4") != std::string::npos ||
                    img.find("pi4") != std::string::npos ||
                    img.find("cm4") != std::string::npos ||
                    img.find("compute module 4") != std::string::npos ||
                    img.find("pi 400") != std::string::npos)
                {
                    return 4;
                }
            }
            if (!n.image.empty()) {
                std::string img = n.image;
                std::transform(img.begin(), img.end(), img.begin(), [](unsigned char c){ return static_cast<char>(std::tolower(c)); });
                if (img.find("2712") != std::string::npos || img.find("rpi5") != std::string::npos || img.find("pi5") != std::string::npos) return 5;
                if (img.find("2711") != std::string::npos || img.find("rpi4") != std::string::npos || img.find("pi4") != std::string::npos) return 4;
            }
            // Fall back to USB product name (e.g., "BCM2712 Boot") seen at connect time
            if (!n.productName.empty()) {
                std::string pn = n.productName;
                std::transform(pn.begin(), pn.end(), pn.begin(), [](unsigned char c){ return static_cast<char>(std::tolower(c)); });
                if (pn.find("2712") != std::string::npos) return 5;
                if (pn.find("2711") != std::string::npos) return 4;
            }
            // Fall back to endpoint-derived hints: if image was populated from manufacturing processor earlier
            if (n.image.empty() && !n.product.empty()) {
                // In some flows we set image to BCM2711/BCM2712 in enrichment; if not present at call time, skip.
            }
            return 0;
        }

        Json::Value topologyToJson(const std::unordered_map<std::string, UsbNode> &nodes,
                                   const std::vector<std::string> &removed = {}) {
            Json::Value root;
            root["type"] = "topology";
            Json::Value arr(Json::arrayValue);
            for (const auto &p : nodes) {
                const UsbNode &n = p.second;
                // Hide root hub interface nodes (e.g., "1-0", "2-0"); we only show their ports as children of server
                if (n.isHub) {
                    const std::string &nid = n.id;
                    if (nid.size() >= 2 && nid.rfind("-0") == nid.size() - 2) {
                        continue;
                    }
                }
                Json::Value j;
                j["id"] = n.id;
                if (!n.parentId.empty()) j["parentId"] = n.parentId; else j["parentId"] = Json::nullValue;
                j["isHub"] = n.isHub;
                if (!n.vendor.empty()) j["vendor"] = n.vendor;
                if (!n.product.empty()) j["product"] = n.product;
                if (!n.productName.empty()) j["productName"] = n.productName;
                if (!n.serial.empty()) j["serial"] = n.serial;
                if (!n.state.empty()) j["state"] = n.state;
                if (!n.image.empty()) j["image"] = n.image;
                if (!n.model.empty()) j["model"] = n.model;
                if (!n.ip.empty()) j["ip"] = n.ip;
                if (n.isPlaceholder) j["placeholder"] = true;
                if (n.speed > 0) j["speed"] = n.speed;
                int gen = inferModelGeneration(n);
                if (gen > 0) j["modelGen"] = gen;
                if (!n.boardType.empty()) j["boardType"] = n.boardType;
                arr.append(j);
            }
            root["nodes"] = arr;
            if (!removed.empty()) {
                Json::Value r(Json::arrayValue);
                for (const auto &id : removed) r.append(id);
                root["removed"] = r;
            }
            auto companionPairs = detectCompanionBuses();
            if (!companionPairs.empty()) {
                Json::Value pairsArr(Json::arrayValue);
                for (const auto &cp : companionPairs) {
                    Json::Value p;
                    p["hsBus"] = cp.hsBus;
                    p["ssBus"] = cp.ssBus;
                    p["ssPortCount"] = cp.ssPortCount;
                    pairsArr.append(p);
                }
                root["companionPairs"] = pairsArr;
            }
            root["timestamp"] = static_cast<Json::UInt64>(std::chrono::duration_cast<std::chrono::milliseconds>(
                std::chrono::system_clock::now().time_since_epoch()).count());
            return root;
        }
        
        void injectTestTopology(const std::string& scenario) {
            std::unordered_map<std::string, UsbNode> newTopology;
            
            if (scenario == "direct") {
                newTopology = generateDirectDevices(4);
            } else if (scenario == "hub") {
                newTopology = generateHubWithDevices(7, 5);
            } else if (scenario == "nested") {
                newTopology = generateNestedHubs(3, 2);
            } else if (scenario == "deep") {
                newTopology = generateNestedHubs(USB_MAX_HUB_DEPTH, 1);
            } else if (scenario == "max") {
                newTopology = generateMaxTopology();
            } else if (scenario == "mixed") {
                newTopology = generateMixedTopology();
            } else if (scenario == "clear" || scenario == "off") {
                testModeEnabled = false;
                LOG_INFO << "Test mode disabled";
                return;
            } else {
                LOG_WARN << "Unknown test scenario: " << scenario;
                return;
            }
            
            {
                std::lock_guard<std::mutex> lock(topologyMutex);
                testTopology = newTopology;
                testModeEnabled = true;
            }
            
            // Broadcast the test topology
            Json::FastWriter w;
            std::string msg = w.write(topologyToJson(newTopology));
            DevicesWebSocketController::broadcast(msg);
            
            LOG_INFO << "Test mode enabled with scenario: " << scenario << " (" << newTopology.size() << " nodes)";
        }

        std::string topologySnapshotString() {
            std::lock_guard<std::mutex> lock(topologyMutex);
            Json::FastWriter w; // compact ok
            return w.write(topologyToJson(currentTopology));
        }
        
        void topologyWorker() {
            LOG_INFO << "Topology worker started";
            Json::Value lastJson;
            while (topologyRunning) {
                // Skip real USB scanning when test mode is enabled
                if (testModeEnabled) {
                    std::unique_lock<std::mutex> lk(scanMu);
                    scanCv.wait_for(lk, std::chrono::seconds(10),
                        [] { return scanRequested || !topologyRunning.load(); });
                    scanRequested = false;
                    continue;
                }
                
                auto newMap = scanUsbSysfs();
                enrichWithProvisioningState(newMap);

                bool changed = false;
                std::vector<std::string> removed;
                {
                    std::lock_guard<std::mutex> lock(topologyMutex);
                    // Simple change detection by size and a few key fields; fallback to full JSON compare
                    if (newMap.size() != currentTopology.size()) {
                        changed = true;
                    } else {
                        for (const auto &p : newMap) {
                            auto it = currentTopology.find(p.first);
                            if (it == currentTopology.end()) { changed = true; break; }
                            const UsbNode &a = p.second;
                            const UsbNode &b = it->second;
                            if (a.parentId != b.parentId || a.vendor != b.vendor || a.product != b.product ||
                                a.serial != b.serial || a.isHub != b.isHub || a.state != b.state ||
                                a.image != b.image || a.ip != b.ip) { changed = true; break; }
                        }
                    }
                    if (changed) {
                        // Compute removed BEFORE updating currentTopology
                        for (const auto &p : currentTopology) {
                            if (p.first == "server") continue;
                            if (newMap.find(p.first) == newMap.end()) removed.push_back(p.first);
                        }
                        // Update current topology after computing removed
                        currentTopology = newMap; // copy to keep newMap for message payload
                    }
                }

                if (changed) {
                    Json::FastWriter w;
                    // Use the updated currentTopology as the payload, along with removed ids
                    std::string msg = w.write(topologyToJson(currentTopology, removed));
                    DevicesWebSocketController::broadcast(msg);
                }

                std::unique_lock<std::mutex> lk(scanMu);
                scanCv.wait_for(lk, std::chrono::seconds(10),
                    [] { return scanRequested || !topologyRunning.load(); });
                scanRequested = false;
            }
            LOG_INFO << "Topology worker stopped";
        }

        // Callback for USB hotplug events from sd_device_monitor: any add/remove/
        // change on the "usb" subsystem wakes the topology worker for an
        // immediate scan. We don't inspect the device — the scanner is the
        // source of truth and will pick up whatever changed.
        int onUsbDeviceEvent(sd_device_monitor *, sd_device *, void *) {
            requestScan();
            return 0;
        }

        // Hosts sd_device_monitor (USB hotplug) in a single sd_event loop.
        // State.db changes used to be observed via inotify on the state
        // directory, but the provisioning scripts now POST to
        // /internal/state-changed after each row write, so the watch is
        // gone — the self-trigger loop with our own SQLite reads goes
        // with it. Shutdown is driven from ~Devices() via sd_event_exit(),
        // which is thread-safe.
        void eventSourceWorker() {
            LOG_INFO << "Event source worker started";

            sd_event *event = nullptr;
            if (sd_event_new(&event) < 0) {
                LOG_ERROR << "sd_event_new failed; falling back to timer-only scanning";
                return;
            }
            eventSourceLoop = event;

            sd_device_monitor *monitor = nullptr;
            if (sd_device_monitor_new(&monitor) < 0) {
                LOG_ERROR << "sd_device_monitor_new failed";
                sd_event_unref(event);
                eventSourceLoop = nullptr;
                return;
            }
            if (sd_device_monitor_filter_add_match_subsystem_devtype(monitor, "usb", nullptr) < 0) {
                LOG_WARN << "sd_device_monitor subsystem filter failed; will receive all uevents";
            }
            if (sd_device_monitor_attach_event(monitor, event) < 0 ||
                sd_device_monitor_start(monitor, onUsbDeviceEvent, nullptr) < 0) {
                LOG_ERROR << "sd_device_monitor_start failed";
                sd_device_monitor_unref(monitor);
                sd_event_unref(event);
                eventSourceLoop = nullptr;
                return;
            }

            sd_event_loop(event);

            sd_device_monitor_unref(monitor);
            sd_event_unref(event);
            eventSourceLoop = nullptr;
            LOG_INFO << "Event source worker stopped";
        }
    }
    
    // Provide a wrapper visible from outside the anonymous namespace
    std::string getTopologySnapshotString() {
        return topologySnapshotString();
    }

    Devices::Devices() 
        :
        systemd_bus(nullptr, sd_bus_unref)
    {
        // Record app start time for gating enrichment to contemporaneous records
        appStartMs = std::chrono::duration_cast<std::chrono::milliseconds>(
            std::chrono::system_clock::now().time_since_epoch()).count();
        sd_bus* bus = nullptr;
        int ret = sd_bus_default_system(&bus);
        if (ret < 0) {
            throw std::runtime_error("Failed to connect to system bus: " + std::string(strerror(-ret)));
        }
        systemd_bus = std::unique_ptr<sd_bus, decltype(&sd_bus_unref)>(bus, sd_bus_unref);
        // Start topology watcher and event-driven scan triggers
        if (!topologyRunning) {
            topologyRunning = true;
            topologyThread = std::thread(topologyWorker);
            eventSourceThread = std::thread(eventSourceWorker);
        }
    }

    Devices::~Devices() {
        // Stop topology watcher and event sources. Order: flip the flag,
        // wake the topology worker via the cv, tell sd_event to exit, then
        // join both threads.
        if (topologyRunning) {
            topologyRunning = false;
            scanCv.notify_all();
            if (eventSourceLoop) sd_event_exit(eventSourceLoop, 0);
            if (topologyThread.joinable()) topologyThread.join();
            if (eventSourceThread.joinable()) eventSourceThread.join();
        }
    }

    // Resolves a URL identifier (which may be a real device serial or a USB
    // endpoint path used before the device reported its serial) to the real
    // device serial recorded in state.db. Returns an empty string when no
    // matching device exists or the device has not yet reported a serial.
    // Special flags are keyed by real serial only — bootstrap.sh looks them
    // up via TARGET_DEVICE_SERIAL — so this resolution is required to keep
    // flag writes and reads aligned with what the bootstrap script honours.
    static std::string resolveDeviceSerial(const std::string &identifier)
    {
        if (identifier.empty()) return {};

        sqlite3* db = nullptr;
        if (sqlite3_open("/srv/rpi-sb-provisioner/state.db", &db) != SQLITE_OK) {
            if (db) sqlite3_close(db);
            return {};
        }
        sqlite3_busy_timeout(db, 5000);

        std::string resolved;
        for (const char* sql : {
                "SELECT serial FROM devices WHERE serial = ? ORDER BY ts DESC, id DESC LIMIT 1",
                "SELECT serial FROM devices WHERE endpoint = ? ORDER BY ts DESC, id DESC LIMIT 1"}) {
            sqlite3_stmt* stmt = nullptr;
            if (sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr) != SQLITE_OK) {
                if (stmt) sqlite3_finalize(stmt);
                continue;
            }
            sqlite3_bind_text(stmt, 1, identifier.c_str(), -1, SQLITE_STATIC);
            if (sqlite3_step(stmt) == SQLITE_ROW) {
                const unsigned char* s = sqlite3_column_text(stmt, 0);
                if (s && *s) {
                    resolved = reinterpret_cast<const char*>(s);
                    sqlite3_finalize(stmt);
                    break;
                }
            }
            sqlite3_finalize(stmt);
        }

        sqlite3_close(db);
        return resolved;
    }

    // Probe for an active special-flag file, honouring the short-vs-full
    // serial representation that rpi-sb-bootstrap.sh's resolve_special_flag_path
    // accepts: when `serial` is a 16-hex-char DUID we also probe the
    // lowercased 8-char suffix, which is the truncated serial form recorded
    // pre-triage. Keep this in lockstep with the shell helper so the UI
    // reflects exactly what the bootstrap script will honour.
    // Returns the matching path, or empty if no flag file is present.
    static std::string findSpecialFlagFile(const std::string &flagId, const std::string &serial)
    {
        if (serial.empty()) return {};
        const std::string dir = "/etc/rpi-sb-provisioner/special-" + flagId + "/";

        std::string sanitised = provisioner::utils::sanitize_path_component(serial);
        if (!sanitised.empty()) {
            std::string p = dir + sanitised;
            if (std::filesystem::is_regular_file(p)) return p;
        }

        if (serial.size() == 16 &&
            std::all_of(serial.begin(), serial.end(),
                        [](unsigned char c) { return std::isxdigit(c) != 0; })) {
            std::string suffix = serial.substr(8);
            std::transform(suffix.begin(), suffix.end(), suffix.begin(),
                           [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
            std::string sanitisedSuffix = provisioner::utils::sanitize_path_component(suffix);
            if (!sanitisedSuffix.empty()) {
                std::string p = dir + sanitisedSuffix;
                if (std::filesystem::is_regular_file(p)) return p;
            }
        }
        return {};
    }

    void Devices::registerHandlers(drogon::HttpAppFramework &app)
    {
        app.registerHandler("/devices", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback) {
            // Add audit log entry for handler access
            AuditLog::logHandlerAccess(req, "/devices");
            
            auto devices = std::make_unique<DeviceList>();
            auto resp = HttpResponse::newHttpResponse();

            sqlite3* db;
            int rc = sqlite3_open("/srv/rpi-sb-provisioner/state.db", &db);
            if (rc) {
                auto errorMsg = "Failed to open database: " + std::string(sqlite3_errmsg(db));
                
                // Check if the client accepts HTML
                auto acceptHeader = req->getHeader("Accept");
                if (!acceptHeader.empty() && (acceptHeader.find("text/html") != std::string::npos)) {
                    auto resp = provisioner::utils::createErrorResponse(
                        req,
                        errorMsg,
                        drogon::k500InternalServerError,
                        "Database Error",
                        "DB_OPEN_ERROR"
                    );
                    callback(resp);
                    return;
                } else {
                    auto resp = provisioner::utils::createErrorResponse(
                        req,
                        errorMsg,
                        drogon::k500InternalServerError,
                        "Database Error",
                        "DB_OPEN_ERROR"
                    );
                    callback(resp);
                    return;
                }
            }
            sqlite3_busy_timeout(db, 5000);

            std::vector<std::string> serials;
            sqlite3_stmt* stmt;
            const char* sql = "SELECT serial, endpoint, state, image, ip_address FROM devices ORDER BY ts DESC, id DESC";
            LOG_INFO << "Executing SQL: " << sql;
            rc = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);

            if (rc != SQLITE_OK) {
                auto errorMsg = "Failed to prepare SQL statement: " + std::string(sqlite3_errmsg(db));
                LOG_ERROR << errorMsg;
                sqlite3_close(db);
                
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    errorMsg,
                    drogon::k500InternalServerError,
                    "Database Error",
                    "SQL_PREPARE_ERROR",
                    std::string(sqlite3_errmsg(db))
                );
                callback(resp);
                return;
            }

            while (sqlite3_step(stmt) == SQLITE_ROW) {
                const unsigned char* serial = sqlite3_column_text(stmt, 0);
                const unsigned char* endpoint = sqlite3_column_text(stmt, 1);
                const unsigned char* state = sqlite3_column_text(stmt, 2);
                const unsigned char* image = sqlite3_column_text(stmt, 3);
                const unsigned char* ip_address = sqlite3_column_text(stmt, 4);
                
                LOG_INFO << "Found device: " << (const char*)serial << ", " << (const char*)endpoint << ", " << (const char*)state;
                
                devices->devices.push_back({
                    (const char*)serial, 
                    (const char*)endpoint, 
                    (const char*)ip_address,
                    (const char*)state,
                    (const char*)image
                });
            }

            sqlite3_finalize(stmt);
            sqlite3_close(db);

            // Check if the client accepts HTML
            auto acceptHeader = req->getHeader("Accept");
            if (!acceptHeader.empty() && (acceptHeader.find("text/html") != std::string::npos)) {
                LOG_INFO << "HTML response requested";

                HttpViewData viewData;
                std::vector<std::map<std::string, std::string>> devicesList;
                for (const auto& device : devices->devices) {
                    std::map<std::string, std::string> deviceMap;
                    deviceMap["serial"] = device.serial;
                    deviceMap["port"] = device.port;
                    deviceMap["ip_address"] = device.ip_address;
                    deviceMap["state"] = device.state;
                    deviceMap["image"] = device.image;
                    devicesList.push_back(deviceMap);
                }
                viewData.insert("devices", devicesList);
                viewData.insert("currentPage", std::string("devices"));
                resp = HttpResponse::newHttpViewResponse("devices.csp", viewData);
            } else {
                // JSON response for API clients
                LOG_INFO << "JSON response requested";
                resp->setStatusCode(k200OK);
                resp->setContentTypeCode(CT_APPLICATION_JSON);

                Json::Value root;
                Json::Value deviceArray(Json::arrayValue);
                for (const auto& device : devices->devices) {
                    Json::Value deviceObj;
                    deviceObj["serial"] = device.serial;
                    deviceObj["port"] = device.port;
                    deviceObj["ip_address"] = device.ip_address;
                    deviceObj["state"] = device.state;
                    deviceObj["image"] = device.image;
                    deviceArray.append(deviceObj);
                }
                root["devices"] = deviceArray;

                Json::FastWriter writer;
                std::string jsonString = writer.write(root);
                resp->setBody(jsonString);
            }

            callback(resp);
        }); // devices handler

        app.registerHandler("/devices/{serialno}", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Device detail request for serial: '" << serialno << "'";
            
            // Add audit log entry for handler access
            AuditLog::logHandlerAccess(req, "/devices/" + serialno);
            
            if (serialno.empty()) {
                LOG_ERROR << "Empty serial number in request";
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Serial number is required",
                    drogon::k400BadRequest,
                    "Missing Parameter",
                    "MISSING_SERIAL"
                );
                callback(resp);
                return;
            }

            sqlite3* db;
            int rc = sqlite3_open("/srv/rpi-sb-provisioner/state.db", &db);
            if (rc) {
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Failed to open device database",
                    drogon::k500InternalServerError,
                    "Database Error",
                    "DB_OPEN_ERROR",
                    std::string(sqlite3_errmsg(db))
                );
                sqlite3_close(db);
                callback(resp);
                return;
            }
            sqlite3_busy_timeout(db, 5000);

            // Try matching by serial first, then fall back to matching by
            // USB endpoint path.  This allows the detail page to be reached
            // via /devices/<usbPath> for devices that don't yet have a serial
            // (e.g. Zero 2 W in RPIBOOT, or early bootstrap phase).
            bool matchedByEndpoint = false;
            sqlite3_stmt* stmt;
            const char* sql = "SELECT serial, endpoint, state, image, ip_address FROM devices WHERE serial = ? ORDER BY ts DESC, id DESC LIMIT 1";
            rc = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);

            if (rc != SQLITE_OK) {
                sqlite3_close(db);
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Failed to prepare SQL statement",
                    drogon::k500InternalServerError,
                    "Database Error",
                    "SQL_PREPARE_ERROR",
                    std::string(sqlite3_errmsg(db))
                );
                callback(resp);
                return;
            }

            sqlite3_bind_text(stmt, 1, serialno.c_str(), -1, SQLITE_STATIC);

            if (sqlite3_step(stmt) != SQLITE_ROW) {
                sqlite3_finalize(stmt);
                LOG_INFO << "Device lookup: no serial match for '" << serialno << "', trying endpoint fallback";
                const char* endpointSql = "SELECT serial, endpoint, state, image, ip_address FROM devices WHERE endpoint = ? ORDER BY ts DESC, id DESC LIMIT 1";
                rc = sqlite3_prepare_v2(db, endpointSql, -1, &stmt, nullptr);
                if (rc != SQLITE_OK) {
                    sqlite3_close(db);
                    auto resp = provisioner::utils::createErrorResponse(
                        req,
                        "Failed to prepare SQL statement",
                        drogon::k500InternalServerError,
                        "Database Error",
                        "SQL_PREPARE_ERROR",
                        std::string(sqlite3_errmsg(db))
                    );
                    callback(resp);
                    return;
                }
                sqlite3_bind_text(stmt, 1, serialno.c_str(), -1, SQLITE_STATIC);
                if (sqlite3_step(stmt) != SQLITE_ROW) {
                    sqlite3_finalize(stmt);
                    sqlite3_close(db);
                    LOG_WARN << "Device lookup: no match for '" << serialno << "' by serial or endpoint";
                    auto resp = provisioner::utils::createErrorResponse(
                        req,
                        "Device not found in database",
                        drogon::k400BadRequest,
                        "Device Not Found",
                        "DEVICE_NOT_FOUND",
                        "Requested identifier: " + serialno
                    );
                    callback(resp);
                    return;
                }
                matchedByEndpoint = true;
                LOG_INFO << "Device lookup: endpoint match for '" << serialno << "'";
            } else {
                LOG_INFO << "Device lookup: serial match for '" << serialno << "'";
            }

            const unsigned char* serial = sqlite3_column_text(stmt, 0);
            const unsigned char* endpoint = sqlite3_column_text(stmt, 1);
            const unsigned char* state = sqlite3_column_text(stmt, 2);
            const unsigned char* image = sqlite3_column_text(stmt, 3);
            const unsigned char* ip_address = sqlite3_column_text(stmt, 4);

            Device device;
            device.serial = serial ? (const char*)serial : "";
            device.port = endpoint ? (const char*)endpoint : "";
            device.ip_address = ip_address ? (const char*)ip_address : "";
            device.state = state ? (const char*)state : "";
            device.image = image ? (const char*)image : "";

            sqlite3_finalize(stmt);
            sqlite3_close(db);

            // Enrich with manufacturing data if available (e.g. os_image_filename for completed devices)
            if (!device.serial.empty()) {
                sqlite3* mdb = nullptr;
                int mrc = sqlite3_open("/srv/rpi-sb-provisioner/manufacturing.db", &mdb);
                if (mrc == SQLITE_OK) {
                    sqlite3_busy_timeout(mdb, 5000);
                    sqlite3_stmt* mstmt = nullptr;
                    const char* msql = "SELECT os_image_filename FROM devices WHERE serial = ? ORDER BY provision_ts DESC LIMIT 1";
                    mrc = sqlite3_prepare_v2(mdb, msql, -1, &mstmt, nullptr);
                    if (mrc == SQLITE_OK) {
                        sqlite3_bind_text(mstmt, 1, device.serial.c_str(), -1, SQLITE_STATIC);
                        if (sqlite3_step(mstmt) == SQLITE_ROW) {
                            const unsigned char* osImage = sqlite3_column_text(mstmt, 0);
                            if (osImage && device.image.empty()) {
                                device.image = reinterpret_cast<const char*>(osImage);
                            }
                        }
                        sqlite3_finalize(mstmt);
                    }
                    sqlite3_close(mdb);
                }
            }

            // Check if the client accepts HTML
            auto acceptHeader = req->getHeader("Accept");
            if (!acceptHeader.empty() && (acceptHeader.find("text/html") != std::string::npos)) {
                // Read log files, trying serial-keyed directory first, then
                // falling back to the endpoint/USB-path-keyed directory.
                // This handles devices where the serial isn't available from the
                // bootloader and we can only identify by USB path.
                std::string provisioner_log, bootstrap_log, triage_log;
                
                std::string serialDir = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(device.serial);
                std::string endpointDir = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(device.port);
                
                auto tryReadLog = [](const std::string &primaryDir, const std::string &fallbackDir, const std::string &filename) -> std::string {
                    for (const auto &dir : {primaryDir, fallbackDir}) {
                        if (dir.empty()) continue;
                        std::string path = dir + "/" + filename;
                        std::ifstream f(path);
                        if (f.is_open()) {
                            AuditLog::logFileSystemAccess("READ", path, true);
                            std::stringstream buf;
                            buf << f.rdbuf();
                            return buf.str();
                        }
                        AuditLog::logFileSystemAccess("READ", path, false);
                    }
                    return {};
                };
                
                provisioner_log = tryReadLog(serialDir, endpointDir, "provisioner.log");
                bootstrap_log = tryReadLog(serialDir, endpointDir, "bootstrap.log");
                triage_log = tryReadLog(serialDir, endpointDir, "triage.log");

                // Query last 10 state changes for this device.
                // When the device was looked up by endpoint (USB path), query
                // history by endpoint too so we get results for devices whose
                // serial may have changed across phases.
                std::vector<std::map<std::string, std::string>> stateHistory;
                {
                    sqlite3* histDb;
                    int histRc = sqlite3_open("/srv/rpi-sb-provisioner/state.db", &histDb);
                    if (histRc == SQLITE_OK) {
                        sqlite3_busy_timeout(histDb, 5000);
                        sqlite3_stmt* histStmt;
                        const char* histSql = matchedByEndpoint
                            ? "SELECT state, ts FROM devices WHERE endpoint = ? ORDER BY ts DESC, id DESC LIMIT 10"
                            : "SELECT state, ts FROM devices WHERE serial = ? ORDER BY ts DESC, id DESC LIMIT 10";
                        histRc = sqlite3_prepare_v2(histDb, histSql, -1, &histStmt, nullptr);
                        if (histRc == SQLITE_OK) {
                            const std::string &histKey = matchedByEndpoint ? device.port : device.serial;
                            sqlite3_bind_text(histStmt, 1, histKey.c_str(), -1, SQLITE_STATIC);
                            while (sqlite3_step(histStmt) == SQLITE_ROW) {
                                const unsigned char* hState = sqlite3_column_text(histStmt, 0);
                                const unsigned char* hTs = sqlite3_column_text(histStmt, 1);
                                std::map<std::string, std::string> entry;
                                entry["state"] = hState ? reinterpret_cast<const char*>(hState) : "";
                                entry["timestamp"] = hTs ? reinterpret_cast<const char*>(hTs) : "";
                                stateHistory.push_back(entry);
                            }
                            sqlite3_finalize(histStmt);
                        }
                        sqlite3_close(histDb);
                    }
                }

                // Read special flag status for this device. Bootstrap.sh
                // accepts a flag file keyed on either the full DUID or its
                // lowercased 8-char truncation, so findSpecialFlagFile probes
                // both forms.
                std::vector<std::map<std::string, std::string>> flagsList;
                struct FlagDef {
                    std::string id;
                    std::string label;
                    std::string description;
                    std::string constraint;
                };
                std::vector<FlagDef> flagDefs = {
                    {"skip-eeprom", "Skip EEPROM Update",
                     "Skips writing the EEPROM/bootloader during bootstrap. Use for devices that already have the correct EEPROM or have been signed.", ""},
                    {"reprovision-device", "Re-provision Device",
                     "Re-provisions an already-provisioned device by re-signing bootcode from original firmware. Only works on Pi 5 family.", "Pi 5 only (BCM2712)"},
                };
                for (const auto &fd : flagDefs) {
                    std::map<std::string, std::string> flagMap;
                    flagMap["id"] = fd.id;
                    flagMap["label"] = fd.label;
                    flagMap["description"] = fd.description;
                    flagMap["constraint"] = fd.constraint;
                    flagMap["active"] = "false";
                    flagMap["mode"] = "off";
                    std::string filePath = findSpecialFlagFile(fd.id, device.serial);
                    if (!filePath.empty()) {
                        flagMap["active"] = "true";
                        std::ifstream ff(filePath);
                        std::string content;
                        if (ff.is_open()) {
                            std::getline(ff, content);
                            while (!content.empty() && (content.back() == '\n' || content.back() == '\r' || content.back() == ' '))
                                content.pop_back();
                        }
                        flagMap["mode"] = (content == "once") ? "once" : "persistent";
                    }
                    flagsList.push_back(flagMap);
                }

                HttpViewData viewData;
                viewData.insert("device.serial", device.serial);
                viewData.insert("device.port", device.port);
                viewData.insert("device.ip_address", device.ip_address);
                viewData.insert("device.state", device.state);
                viewData.insert("device.image", device.image);
                viewData.insert("provisioner_log", provisioner_log);
                viewData.insert("bootstrap_log", bootstrap_log);
                viewData.insert("triage_log", triage_log);
                viewData.insert("flags", flagsList);
                viewData.insert("stateHistory", stateHistory);
                viewData.insert("currentPage", std::string("devices"));
                resp = HttpResponse::newHttpViewResponse("device_detail.csp", viewData);
            } else {
                Json::Value root;
                root["serial"] = device.serial;
                root["port"] = device.port;
                root["state"] = device.state;

                Json::FastWriter writer;
                std::string jsonString = writer.write(root);
                resp->setStatusCode(k200OK);
                resp->setContentTypeCode(CT_APPLICATION_JSON);
                resp->setBody(jsonString);
            }

            callback(resp);
        }); // devices/{serialno} handler

        app.registerHandler("/devices/{serialno}/log/provisioner", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Provisioner log request for serial: '" << serialno << "'";
            
            // Add audit log entry for handler access
            AuditLog::logHandlerAccess(req, "/devices/" + serialno + "/log/provisioner");
            
            if (serialno.empty()) {
                LOG_ERROR << "Empty serial number in request";
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Serial number is required",
                    drogon::k400BadRequest,
                    "Missing Parameter",
                    "MISSING_SERIAL"
                );
                callback(resp);
                return;
            }

            std::string logPath = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(serialno) + "/provisioner.log";
            LOG_INFO << "Attempting to open log file at: " << logPath;
            
            std::ifstream logFile(logPath);
            if (!logFile.is_open()) {
                LOG_ERROR << "Failed to open log file: " << logPath;
                // Log failed file access to audit log
                AuditLog::logFileSystemAccess("READ", logPath, false);
                
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Log file not found",
                    drogon::k400BadRequest,
                    "Log Not Found",
                    "LOG_NOT_FOUND",
                    "Attempted path: " + logPath
                );
                callback(resp);
                return;
            }

            // Log successful file access to audit log
            AuditLog::logFileSystemAccess("READ", logPath, true);
            
            LOG_INFO << "Successfully opened log file: " << logPath;
            std::stringstream buffer;
            buffer << logFile.rdbuf();
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_TEXT_PLAIN);
            resp->setBody(buffer.str());
            callback(resp);
        }); // devices/{serialno}/log/provisioner handler

        app.registerHandler("/devices/{serialno}/log/bootstrap", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Bootstrap log request for serial: '" << serialno << "'";
            
            if (serialno.empty()) {
                LOG_ERROR << "Empty serial number in request";
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Serial number is required",
                    drogon::k400BadRequest,
                    "Missing Parameter",
                    "MISSING_SERIAL"
                );
                callback(resp);
                return;
            }

            std::string logPath = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(serialno) + "/bootstrap.log";
            LOG_INFO << "Attempting to open log file at: " << logPath;
            
            std::ifstream logFile(logPath);
            if (!logFile.is_open()) {
                LOG_ERROR << "Failed to open log file: " << logPath;
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Log file not found",
                    drogon::k400BadRequest,
                    "Log Not Found",
                    "LOG_NOT_FOUND",
                    "Attempted path: " + logPath
                );
                callback(resp);
                return;
            }

            LOG_INFO << "Successfully opened log file: " << logPath;
            std::stringstream buffer;
            buffer << logFile.rdbuf();
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_TEXT_PLAIN);
            resp->setBody(buffer.str());
            callback(resp);
        }); // devices/{serialno}/log/bootstrap handler

        app.registerHandler("/devices/{serialno}/log/triage", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Triage log request for serial: '" << serialno << "'";
            
            if (serialno.empty()) {
                LOG_ERROR << "Empty serial number in request";
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Serial number is required",
                    drogon::k400BadRequest,
                    "Missing Parameter",
                    "MISSING_SERIAL"
                );
                callback(resp);
                return;
            }

            std::string logPath = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(serialno) + "/triage.log";
            LOG_INFO << "Attempting to open log file at: " << logPath;
            
            std::ifstream logFile(logPath);
            if (!logFile.is_open()) {
                LOG_ERROR << "Failed to open log file: " << logPath;
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Log file not found",
                    drogon::k400BadRequest,
                    "Log Not Found",
                    "LOG_NOT_FOUND",
                    "Attempted path: " + logPath
                );
                callback(resp);
                return;
            }

            LOG_INFO << "Successfully opened log file: " << logPath;
            std::stringstream buffer;
            buffer << logFile.rdbuf();
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_TEXT_PLAIN);
            resp->setBody(buffer.str());
            callback(resp);
        }); // devices/:serialno/log/triage handler

        app.registerHandler("/devices/{serialno}/key/public", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Public key request for serial: '" << serialno << "'";
            
            if (serialno.empty()) {
                LOG_ERROR << "Empty serial number in request";
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Serial number is required",
                    drogon::k400BadRequest,
                    "Missing Parameter",
                    "MISSING_SERIAL"
                );
                callback(resp);
                return;
            }

            std::string keyPath = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(serialno) + "/keypair/" + provisioner::utils::sanitize_path_component(serialno) + ".pub";
            std::ifstream keyFile(keyPath);
            if (!keyFile.is_open()) {
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Public key file not found",
                    drogon::k400BadRequest,
                    "Key Not Found",
                    "KEY_NOT_FOUND",
                    "Attempted path: " + keyPath
                );
                callback(resp);
                return;
            }

            std::stringstream buffer;
            buffer << keyFile.rdbuf();
            resp->setContentTypeCode(CT_APPLICATION_OCTET_STREAM);
            resp->setStatusCode(k200OK);
            resp->setBody(buffer.str());
            callback(resp);
        }); // devices/{serialno}/key/public handler

        app.registerHandler("/devices/{serialno}/key/private", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_WARN << "SECURITY: Private key download request for serial: '" << serialno << "' from " << AuditLog::getClientIP(req);
            
            // SECURITY CHECK: This endpoint is disabled by default for security reasons
            // Private keys should NEVER be exposed via HTTP in production environments
            auto privateKeyAccessEnabled = provisioner::utils::getConfigValue("RPI_SB_PROVISIONER_ENABLE_PRIVATE_KEY_API");
            if (!privateKeyAccessEnabled || *privateKeyAccessEnabled != "true") {
                LOG_ERROR << "SECURITY: Private key API access denied - endpoint is disabled. "
                          << "Set RPI_SB_PROVISIONER_ENABLE_PRIVATE_KEY_API=true in /etc/rpi-sb-provisioner/config to enable this DANGEROUS endpoint.";
                
                // Add audit log entry for denied access attempt
                AuditLog::logHandlerAccess(req, "/devices/" + serialno + "/key/private");
                AuditLog::logFileSystemAccess("DENIED_PRIVATE_KEY_ACCESS", 
                    "/var/log/rpi-sb-provisioner/" + serialno + "/keypair/" + serialno + ".der", 
                    false, 
                    "", 
                    "Private key API is disabled for security. Client IP: " + AuditLog::getClientIP(req));
                
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Private key download API is disabled for security reasons. "
                    "This endpoint must be explicitly enabled in configuration: "
                    "Set RPI_SB_PROVISIONER_ENABLE_PRIVATE_KEY_API=true in /etc/rpi-sb-provisioner/config. "
                    "WARNING: Enabling this endpoint exposes device private keys via HTTP and should only be done in secure, isolated networks.",
                    drogon::k403Forbidden,
                    "Endpoint Disabled",
                    "PRIVATE_KEY_API_DISABLED",
                    "This is a security feature. Private keys are cryptographic secrets that should not be transmitted over HTTP."
                );
                callback(resp);
                return;
            }
            
            // Log that this dangerous endpoint is enabled and being used
            LOG_WARN << "SECURITY WARNING: Private key API is ENABLED and being accessed. "
                     << "This is a security risk. Private keys are being transmitted over HTTP.";
            
            // Add audit log entry for handler access
            AuditLog::logHandlerAccess(req, "/devices/" + serialno + "/key/private");
            
            if (serialno.empty()) {
                LOG_ERROR << "Empty serial number in request";
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Serial number is required",
                    drogon::k400BadRequest,
                    "Missing Parameter",
                    "MISSING_SERIAL"
                );
                callback(resp);
                return;
            }

            std::string keyPath = "/var/log/rpi-sb-provisioner/" + provisioner::utils::sanitize_path_component(serialno) + "/keypair/" + provisioner::utils::sanitize_path_component(serialno) + ".der";
            std::ifstream keyFile(keyPath);
            if (!keyFile.is_open()) {
                // Log failed file access to audit log
                AuditLog::logFileSystemAccess("READ_PRIVATE_KEY", keyPath, false, "", 
                    "Private key file not found for serial: " + serialno + ", Client IP: " + AuditLog::getClientIP(req));
                
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Private key file not found",
                    drogon::k400BadRequest,
                    "Key Not Found",
                    "KEY_NOT_FOUND",
                    "Attempted path: " + keyPath
                );
                callback(resp);
                return;
            }

            // CRITICAL SECURITY LOG: Private key is being transmitted
            LOG_WARN << "SECURITY ALERT: Private key for device " << serialno 
                     << " is being transmitted to " << AuditLog::getClientIP(req);
            
            // Log successful private key access to audit log
            AuditLog::logFileSystemAccess("READ_PRIVATE_KEY", keyPath, true, "", 
                "CRITICAL: Private key transmitted for serial: " + serialno + 
                ", Client IP: " + AuditLog::getClientIP(req) + 
                ", User-Agent: " + req->getHeader("User-Agent"));

            std::stringstream buffer;
            buffer << keyFile.rdbuf();
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_APPLICATION_OCTET_STREAM);
            resp->setBody(buffer.str());
            callback(resp);
        }); // devices/{serialno}/key/private handler

        // Secret test mode endpoint - only accessible with special header or localhost
        app.registerHandler("/devices/_test/{scenario}", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &scenario) {
            auto resp = HttpResponse::newHttpResponse();
            
            // Security: Only allow from localhost or with secret header
            std::string clientIP = AuditLog::getClientIP(req);
            std::string secretHeader = req->getHeader("X-Test-Secret");
            bool isLocalhost = (clientIP == "127.0.0.1" || clientIP == "::1" || clientIP.find("localhost") != std::string::npos);
            bool hasSecret = (secretHeader == "rpi-provisioner-test-2024");
            
            if (!isLocalhost && !hasSecret) {
                resp->setStatusCode(k404NotFound);
                resp->setBody("Not Found");
                callback(resp);
                return;
            }
            
            LOG_INFO << "Test mode request: scenario=" << scenario << " from " << clientIP;
            
            // Inject the test topology
            injectTestTopology(scenario);
            
            Json::Value result;
            result["status"] = "ok";
            result["scenario"] = scenario;
            result["testMode"] = testModeEnabled.load();
            result["availableScenarios"] = Json::arrayValue;
            result["availableScenarios"].append("direct");   // 4 devices directly on root ports
            result["availableScenarios"].append("hub");      // 7-port hub with 5 devices
            result["availableScenarios"].append("nested");   // 3-level nested hubs
            result["availableScenarios"].append("deep");     // Max depth (5 hubs) chain
            result["availableScenarios"].append("max");      // Near 127 device limit
            result["availableScenarios"].append("mixed");    // Mixed topology
            result["availableScenarios"].append("clear");    // Disable test mode
            result["availableScenarios"].append("off");      // Alias for clear
            
            Json::FastWriter writer;
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_APPLICATION_JSON);
            resp->setBody(writer.write(result));
            callback(resp);
        }); // devices/_test/{scenario} handler

        // ===== Special Flags Management =====
        // Known special flags and their descriptions
        struct SpecialFlagInfo {
            std::string id;           // directory name suffix (e.g., "skip-eeprom")
            std::string label;        // human-readable label
            std::string description;  // what this flag does
            std::string constraint;   // any device constraint (empty = all devices)
        };

        static const std::vector<SpecialFlagInfo> knownFlags = {
            {"skip-eeprom",
             "Skip EEPROM Update",
             "Skips writing the EEPROM/bootloader during the bootstrap phase. "
             "Use this for devices that already have the correct EEPROM configuration, "
             "or devices that have already been signed.",
             ""},
            {"reprovision-device",
             "Re-provision Device",
             "Re-provisions a device that has already been provisioned. "
             "This re-signs the bootcode using the original firmware recovery.bin "
             "instead of the cached signed bootcode. Only works on Raspberry Pi 5 family devices.",
             "Pi 5 only (BCM2712)"},
        };

        // GET /devices/{serialno}/flags - Read current flag status
        app.registerHandler("/devices/{serialno}/flags",
            [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Flags request for serial: '" << serialno << "'";

            AuditLog::logHandlerAccess(req, "/devices/" + serialno + "/flags");

            if (serialno.empty()) {
                auto resp = provisioner::utils::createErrorResponse(
                    req, "Serial number is required",
                    drogon::k400BadRequest, "Missing Parameter", "MISSING_SERIAL");
                callback(resp);
                return;
            }

            // The URL identifier may be a USB endpoint path for devices that
            // haven't reported their serial yet. Resolve to the real serial,
            // then probe both the full and truncated forms via
            // findSpecialFlagFile so the response matches what bootstrap.sh
            // will honour at flag-evaluation time.
            std::string realSerial = resolveDeviceSerial(serialno);

            Json::Value root;
            Json::Value flagsArray(Json::arrayValue);

            for (const auto &fi : knownFlags) {
                Json::Value flag;
                flag["id"] = fi.id;
                flag["label"] = fi.label;
                flag["description"] = fi.description;
                if (!fi.constraint.empty()) flag["constraint"] = fi.constraint;
                flag["active"] = false;
                flag["mode"] = "off";

                std::string filePath = findSpecialFlagFile(fi.id, realSerial);
                if (!filePath.empty()) {
                    flag["active"] = true;
                    std::ifstream f(filePath);
                    std::string content;
                    if (f.is_open()) {
                        std::getline(f, content);
                        while (!content.empty() && (content.back() == '\n' || content.back() == '\r' || content.back() == ' '))
                            content.pop_back();
                    }
                    flag["mode"] = (content == "once") ? "once" : "persistent";
                }

                flagsArray.append(flag);
            }

            root["serial"] = realSerial.empty() ? serialno : realSerial;
            root["flags"] = flagsArray;

            Json::FastWriter writer;
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_APPLICATION_JSON);
            resp->setBody(writer.write(root));
            callback(resp);
        }, {Get});

        // POST /devices/{serialno}/flags - Set a flag
        app.registerHandler("/devices/{serialno}/flags",
            [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, const std::string &serialno) {
            auto resp = HttpResponse::newHttpResponse();
            LOG_INFO << "Set flag request for serial: '" << serialno << "'";

            AuditLog::logHandlerAccess(req, "/devices/" + serialno + "/flags");

            if (serialno.empty()) {
                auto resp = provisioner::utils::createErrorResponse(
                    req, "Serial number is required",
                    drogon::k400BadRequest, "Missing Parameter", "MISSING_SERIAL");
                callback(resp);
                return;
            }

            auto body = req->getJsonObject();
            if (!body) {
                auto resp = provisioner::utils::createErrorResponse(
                    req, "Invalid JSON request body",
                    drogon::k400BadRequest, "Invalid Request", "INVALID_JSON");
                callback(resp);
                return;
            }

            std::string flagId = body->get("flag", "").asString();
            std::string mode = body->get("mode", "").asString();

            if (flagId.empty()) {
                auto resp = provisioner::utils::createErrorResponse(
                    req, "Flag name is required",
                    drogon::k400BadRequest, "Missing Parameter", "MISSING_FLAG");
                callback(resp);
                return;
            }

            if (mode != "once" && mode != "persistent" && mode != "off") {
                auto resp = provisioner::utils::createErrorResponse(
                    req, "Mode must be 'once', 'persistent', or 'off'",
                    drogon::k400BadRequest, "Invalid Parameter", "INVALID_MODE");
                callback(resp);
                return;
            }

            // Validate flag is a known flag
            bool validFlag = false;
            for (const auto &fi : knownFlags) {
                if (fi.id == flagId) { validFlag = true; break; }
            }
            if (!validFlag) {
                auto resp = provisioner::utils::createErrorResponse(
                    req, "Unknown flag: " + flagId,
                    drogon::k400BadRequest, "Invalid Parameter", "UNKNOWN_FLAG");
                callback(resp);
                return;
            }

            // Flags are honoured by bootstrap.sh keyed on the real device
            // serial. If the request came in via a USB endpoint URL (used
            // before the device has reported its serial), resolve it so the
            // file we write is one the bootstrap script will actually read.
            std::string realSerial = resolveDeviceSerial(serialno);
            if (realSerial.empty()) {
                auto resp = provisioner::utils::createErrorResponse(
                    req,
                    "Device has not reported a serial yet; flag cannot be set until it does. "
                    "Wait for the device to progress past the initial bootloader stage and retry.",
                    drogon::k409Conflict, "Serial Not Available", "SERIAL_NOT_RESOLVED",
                    "Requested identifier: " + serialno);
                callback(resp);
                return;
            }

            std::string sanitizedSerial = provisioner::utils::sanitize_path_component(realSerial);
            std::string dirPath = "/etc/rpi-sb-provisioner/special-" + flagId;
            std::string filePath = dirPath + "/" + sanitizedSerial;

            if (mode == "off") {
                // Remove the flag file
                if (std::filesystem::exists(filePath)) {
                    try {
                        std::filesystem::remove(filePath);
                        AuditLog::logFileSystemAccess("DELETE_FLAG", filePath, true, "",
                            "Special flag removed: " + flagId + " for device " + realSerial);
                        LOG_INFO << "Removed special flag: " << flagId << " for device " << realSerial;
                    } catch (const std::exception &e) {
                        LOG_ERROR << "Failed to remove flag file: " << e.what();
                        auto resp = provisioner::utils::createErrorResponse(
                            req, "Failed to remove flag file",
                            drogon::k500InternalServerError, "File Error", "FLAG_REMOVE_ERROR");
                        callback(resp);
                        return;
                    }
                }
            } else {
                // Create directory if it doesn't exist
                try {
                    if (!std::filesystem::exists(dirPath)) {
                        std::filesystem::create_directories(dirPath);
                    }
                } catch (const std::exception &e) {
                    LOG_ERROR << "Failed to create flag directory: " << e.what();
                    auto resp = provisioner::utils::createErrorResponse(
                        req, "Failed to create flag directory",
                        drogon::k500InternalServerError, "File Error", "FLAG_DIR_ERROR");
                    callback(resp);
                    return;
                }

                // Write the flag file with mode content
                std::ofstream f(filePath);
                if (!f.is_open()) {
                    LOG_ERROR << "Failed to write flag file: " << filePath;
                    auto resp = provisioner::utils::createErrorResponse(
                        req, "Failed to write flag file",
                        drogon::k500InternalServerError, "File Error", "FLAG_WRITE_ERROR");
                    callback(resp);
                    return;
                }
                f << mode << "\n";
                f.close();

                AuditLog::logFileSystemAccess("SET_FLAG", filePath, true, "",
                    "Special flag set: " + flagId + " (" + mode + ") for device " + realSerial);
                LOG_INFO << "Set special flag: " << flagId << " (" << mode << ") for device " << realSerial;
            }

            // Return the updated flag status. Report the real serial so the
            // client can reconcile against /devices/{serial}.
            Json::Value result;
            result["success"] = true;
            result["flag"] = flagId;
            result["mode"] = mode;
            result["serial"] = realSerial;

            Json::FastWriter writer;
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_APPLICATION_JSON);
            resp->setBody(writer.write(result));
            callback(resp);
        }, {Post});

        // Test mode status endpoint
        app.registerHandler("/devices/_test", [](const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback) {
            auto resp = HttpResponse::newHttpResponse();
            
            // Security: Only allow from localhost or with secret header
            std::string clientIP = AuditLog::getClientIP(req);
            std::string secretHeader = req->getHeader("X-Test-Secret");
            bool isLocalhost = (clientIP == "127.0.0.1" || clientIP == "::1" || clientIP.find("localhost") != std::string::npos);
            bool hasSecret = (secretHeader == "rpi-provisioner-test-2024");
            
            if (!isLocalhost && !hasSecret) {
                resp->setStatusCode(k404NotFound);
                resp->setBody("Not Found");
                callback(resp);
                return;
            }
            
            Json::Value result;
            result["testMode"] = testModeEnabled.load();
            result["availableScenarios"] = Json::arrayValue;
            result["availableScenarios"].append("direct");
            result["availableScenarios"].append("hub");
            result["availableScenarios"].append("nested");
            result["availableScenarios"].append("deep");
            result["availableScenarios"].append("max");
            result["availableScenarios"].append("mixed");
            result["availableScenarios"].append("clear");
            result["availableScenarios"].append("off");
            result["description"] = Json::objectValue;
            result["description"]["direct"] = "4 devices connected directly to root ports";
            result["description"]["hub"] = "Single 7-port hub with 5 devices";
            result["description"]["nested"] = "3-level nested hub hierarchy";
            result["description"]["deep"] = "Maximum depth (5 external hubs) chain";
            result["description"]["max"] = "Near USB maximum (127 devices)";
            result["description"]["mixed"] = "Mixed topology with direct, hub, and nested devices";
            result["description"]["clear"] = "Disable test mode, return to real USB scanning";
            
            Json::FastWriter writer;
            resp->setStatusCode(k200OK);
            resp->setContentTypeCode(CT_APPLICATION_JSON);
            resp->setBody(writer.write(result));
            callback(resp);
        }); // devices/_test handler

        // Localhost-only notification endpoints. The bash provisioning
        // scripts POST here after writing rows that the topology worker
        // cares about, in place of the inotify watch we used to keep on
        // /srv/rpi-sb-provisioner. Inverting the dependency removes the
        // self-trigger loop the inotify watch had with our own SQLite reads.
        // Both endpoints share the same effect (kick the worker for a
        // rescan); the split is for semantic clarity at the call site and
        // in access logs. No body is required — the worker re-reads both
        // DBs on the next scan, which requestScan() wakes immediately.
        //
        // Access control is two layers:
        //   1. Transport-level loopback check via getPeerAddr().isLoopbackIp().
        //      We must NOT use AuditLog::getClientIP here -- that one honours
        //      X-Forwarded-For, which is attacker-controlled on a direct
        //      connection and would let any reachable client spoof loopback.
        //   2. Shared-secret token in /run/rpi-sb-provisioner/internal.token.
        //      The file is mode 0600, root-owned; only callers in the same
        //      security domain as the provisioning scripts can read it. This
        //      closes the nginx-reverse-proxy hole (peer IP would be 127.0.0.1
        //      even for external clients) and any local-non-root caller.
        // Token: generated once per service start; lives in tmpfs, so a
        // fresh value comes up on every boot.
        const std::string internalToken = [] {
            std::string hex;
            hex.reserve(64);
            std::random_device rd;
            for (int i = 0; i < 32; ++i) {
                const unsigned byte = rd() & 0xff;
                static const char *digits = "0123456789abcdef";
                hex.push_back(digits[byte >> 4]);
                hex.push_back(digits[byte & 0x0f]);
            }
            const std::string tokenPath = "/run/rpi-sb-provisioner/internal.token";
            std::filesystem::create_directories("/run/rpi-sb-provisioner");
            const std::string tmpPath = tokenPath + ".tmp";
            {
                std::ofstream f(tmpPath, std::ios::trunc | std::ios::binary);
                if (f) f << hex;
            }
            // Mode 0600 before rename so a reader never sees a too-permissive file.
            (void)::chmod(tmpPath.c_str(), 0600);
            (void)::rename(tmpPath.c_str(), tokenPath.c_str());
            return hex;
        }();

        // Constant-time string compare (avoid timing-channel leak on the
        // token). std::memcmp is famously *not* constant time on most libcs.
        auto tokensEqual = [](const std::string &a, const std::string &b) {
            if (a.size() != b.size()) return false;
            unsigned char acc = 0;
            for (size_t i = 0; i < a.size(); ++i) {
                acc |= static_cast<unsigned char>(a[i]) ^ static_cast<unsigned char>(b[i]);
            }
            return acc == 0;
        };

        auto internalNotify = [internalToken, tokensEqual](
                const HttpRequestPtr &req,
                std::function<void(const HttpResponsePtr &)> &&callback) {
            auto resp = HttpResponse::newHttpResponse();
            // Layer 1: transport-level loopback. Reject before we even
            // look at headers so spoofed X-Forwarded-For cannot influence
            // the decision. Trantor's isLoopbackIp() refuses v4-mapped IPv6
            // forms (::ffff:127.0.0.1) that show up on dual-stack listeners,
            // so compare the stringified peer address explicitly.
            const std::string peerIp = req->getPeerAddr().toIp();
            const bool isLoopback = (peerIp == "127.0.0.1" || peerIp == "::1" ||
                                     peerIp == "::ffff:127.0.0.1");
            if (!isLoopback) {
                resp->setStatusCode(k404NotFound);
                resp->setBody("debug: non-loopback peer=" + peerIp);
                callback(resp);
                return;
            }
            // Layer 2: shared-secret token from the root-only file.
            const std::string presented = req->getHeader("X-Internal-Token");
            if (presented.empty() || !tokensEqual(presented, internalToken)) {
                resp->setStatusCode(k404NotFound);
                resp->setBody("debug: token mismatch presented_len=" +
                              std::to_string(presented.size()) +
                              " expected_len=" + std::to_string(internalToken.size()));
                callback(resp);
                return;
            }
            requestScan();
            resp->setStatusCode(k204NoContent);
            callback(resp);
        };
        // Paired with record_state() in host-support/state-recording.
        app.registerHandler("/internal/state-changed", internalNotify, {Post});
        // Paired with the manufacturing.db INSERT in host-support/manufacturing-data.
        app.registerHandler("/internal/manufacturing-recorded", internalNotify, {Post});
    }


} // namespace provisioner

