String line;
            final RobotsTxt robotsTxt = new RobotsTxt();
            final List<Directive> currentDirectiveList = new ArrayList<>();
            boolean isGroupRecordStarted = false;
            while ((line = reader.readLine()) != null) {
                line = stripComment(line).trim();
                if (StringUtil.isEmpty(line)) {
                    continue;
                }

        sitemapSet.setType(SitemapSet.URLSET);

        try {
            final BufferedReader br = new BufferedReader(new InputStreamReader(in, Constants.UTF_8));
            String line;
            while ((line = br.readLine()) != null) {
                final String url = line.trim();
                if (StringUtil.isNotBlank(url) && (url.startsWith("http://") || url.startsWith("https://"))) {

            final Map<String, Object>[] array = new Map[(int) response.getHits().getTotalHits().value()];

            int count = 0;
            try {
                while (scrollId != null) {
                    final SearchHit[] hits = response.getHits().getHits();
                    if (hits.length == 0) {
                        break;
                    }

                this.getClass().getClassLoader().getResourceAsStream("suggest_indices/suggest/mappings-default.json")))) {

            String line;
            while ((line = br.readLine()) != null) {
                mappingSource.append(line);
            }
        }
        return mappingSource.toString();
    }

        if (listener != null) {
            listener.onRequestStart(this, request);
        }

        List<Exception> exceptionList = null;
        try {
            int count = 0;
            while (count < maxRetryCount) {
                if (listener != null) {
                    listener.onRequest(this, request, count);
                }

                try {

                BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dest))) {
            final byte[] buf = new byte[1024];
            int length;
            while (-1 < (length = in.read(buf))) {
                out.write(buf, 0, length);
                out.flush();
            }
        } catch (final IOException e) {
            throw new IORuntimeException(e);
        }

            int alphanumSize = 0;
            int symbolSize = 0;
            int c;
            final Set<String> termCache = new HashSet<>(1000);
            try {
                while ((c = reader.read()) != -1) {
                    if (Character.isISOControl(c) || isSpaceChar(c)) {
                        if (duplicateTermRemoved) {
                            if (alphanumSize > 0) {

     * @param func The function to execute.
     * @return The response from the operation.
     */
    public <T> T get(final Function<FesenClient, ActionFuture<T>> func) {
        int retryCount = 0;
        while (true) {
            try {
                return func.apply(this).actionGet(connTimeout, TimeUnit.MILLISECONDS);
            } catch (final IndexNotFoundException | VersionConflictEngineException e) {

                .setSize(scrollSize)
                .execute());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {
                final SearchHits searchHits = response.getHits();
                if (searchHits.getHits().length == 0) {
                    break;
                }

     *
     * @return the response body as an InputStream, or null if no response body is available
     * @throws IORuntimeException if an I/O error occurs while reading the response body file
     */
    public InputStream getResponseBody() {
        if (responseBodyBytes != null) {
            return new ByteArrayInputStream(responseBodyBytes);
        }