// Only throw exception for fatal I/O errors
            throw new RobotsTxtException("Failed to read robots.txt due to I/O error.", e);
        } catch (final Exception e) {
            // Catch any other fatal errors (e.g., encoding issues)
            throw new RobotsTxtException("Failed to parse robots.txt.", e);
        }
    }

    /**