labels.crawling_info_thread_dump=Vidage de thread
labels.crawling_info_CrawlerStartTime=Heure de début du robot d'exploration
labels.crawling_info_CrawlerEndTime=Heure de fin du robot d'exploration
labels.crawling_info_CrawlerExecTime=Temps d'exécution du robot d'exploration
labels.crawling_info_CrawlerStatus=Statut du robot d'exploration
labels.crawling_info_WebFsCrawlExecTime=Temps d'exécution de l'exploration (Web/Fichier)

            }
        }
        if (logger.isDebugEnabled()) {
            logger.debug("FTP client closed");
        }
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.robot.client.S2RobotClient#doGet(java.lang.String)
     */
    @Override
    public ResponseData doGet(final String uri) {
        return processRequest(uri, true);
    }

    /**

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

- **Testing**: JUnit 4, UTFlute, Mockito 5.7.0
- **Storage**: In-memory (default), OpenSearch (optional)

### Protocols

- **HTTP/HTTPS**: Full crawling, cookies, auth, robots.txt
- **File**: Local/network file systems
- **FTP**: With authentication
- **SMB/CIFS**: Windows shares (SMB1/SMB2+)
- **Storage**: MinIO/S3-compatible

### Content Formats

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

      @Override
      long treeAggregate(@Nullable AvlNode<?> root) {
        return (root == null) ? 0 : root.totalCount;
      }
    },
    DISTINCT {
      @Override
      int nodeAggregate(AvlNode<?> node) {
        return 1;
      }

      @Override
      long treeAggregate(@Nullable AvlNode<?> root) {
        return (root == null) ? 0 : root.distinctElements;
      }
    };

starting with its own and including everything up-to but not including the root. We don't need to
include root certificates because the client already has them.

```java
HandshakeCertificates serverHandshakeCertificates = new HandshakeCertificates.Builder()
    .heldCertificate(serverCertificate, intermediateCertificate.certificate())
    .build();
```

The client only needs to know the trusted root certificate. It checks the server's certificate by

      @Override
      long treeAggregate(@Nullable AvlNode<?> root) {
        return (root == null) ? 0 : root.totalCount;
      }
    },
    DISTINCT {
      @Override
      int nodeAggregate(AvlNode<?> node) {
        return 1;
      }

      @Override
      long treeAggregate(@Nullable AvlNode<?> root) {
        return (root == null) ? 0 : root.distinctElements;
      }
    };