*/
    protected void processAccessControlEntries(final ResponseData responseData, final SmbFile file) {
        try {
            final ACE[] aces = file.getSecurity(resolveSids);
            if (aces != null) {
                final Set<SID> sidAllowSet = new HashSet<>();
                final Set<SID> sidDenySet = new HashSet<>();
                for (final ACE ace : aces) {
                    if (logger.isDebugEnabled()) {

     * java.util.List)
     */
    @Override
    public void offerAll(final String sessionId, final List<UrlQueueImpl<Long>> newUrlQueueList) {
        final Queue<UrlQueueImpl<Long>> urlQueueList = dataHelper.getUrlQueueList(sessionId);
        synchronized (urlQueueList) {
            final List<UrlQueueImpl<Long>> targetList = new ArrayList<>();
            for (final UrlQueueImpl<Long> urlQueue : newUrlQueueList) {

    private static final Logger logger = LogManager.getLogger(FileSystemClient.class);

    /** Key for file attribute view in metadata */
    public static final String FILE_ATTRIBUTE_VIEW = "fileAttributeView";

    /** Key for file user in metadata */
    public static final String FS_FILE_USER = "fsFileUser";

    /** Key for file groups in metadata */
    public static final String FS_FILE_GROUPS = "fsFileGroups";

     * @param encoding the encoding of the child URLs
     */
    protected void storeChildUrls(final CrawlerContext crawlerContext, final Set<RequestData> childUrlList, final String url,
            final int depth, final String encoding) {
        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()

     * @return The parameter value, or the default value if not found.
     */
    protected <T> T getInitParameter(final String key, final T defaultValue, final Class<T> clazz) {
        if (initParamMap != null) {
            try {
                final Object paramValue = initParamMap.get(key);
                if (paramValue == null) {
                    return defaultValue;
                }

     * @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.
     * @param pos The position to add the client.
     */
    public void addClient(final String regex, final CrawlerClient client, final int pos) {
        if (StringUtil.isBlank(regex)) {
            throw new CrawlerSystemException("A regular expression is null.");
        }
        if (client == null) {

 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */
    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    /** List of sitemap URLs found in the robots.txt file. */
    private final List<String> sitemapList = new ArrayList<>();

    /**

        final CrawlerWebServer server2 = new CrawlerWebServer(7071);
        server2.start();

        final String url1 = "http://localhost:7070/";
        final String url2 = "http://localhost:7071/";
        try {
            final int maxCount = 10;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();
            file.mkdirs();

    /** Logger instance for this class */
    private static final Logger logger = LogManager.getLogger(ExtractorBuilder.class);

    /** Input stream to extract data from */
    private final InputStream in;

    /** Parameters for the extraction process */
    private final Map<String, String> params;

    /** Container for accessing crawler components */
    private final CrawlerContainer crawlerContainer;

        }
        final Extractor extractor = extractorFactory.getExtractor(responseData.getMimeType());
        final Map<String, String> params = new HashMap<>();
        params.put(ExtractData.RESOURCE_NAME_KEY, getResourceName(responseData));
        params.put(ExtractData.CONTENT_TYPE, responseData.getMimeType());
        String content = null;
        try (final InputStream in = responseData.getResponseBody()) {