}

    public void test_doGet_dir() {
        final File file = ResourceUtil.getResourceAsFile("test");
        String path = file.getAbsolutePath();
        if (!path.startsWith("/")) {
            path = "/" + path.replace('\\', '/');
        }
        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();

        String input = content.replaceAll("[\\r\\n]", " ");
        if (ignoreCommentTag) {
            input = input.replaceAll("<!--[^>]+-->", "");
        } else {
            input = input.replace("<!--", "").replace("-->", "");
        }
        final Matcher matcher = getTagPattern().matcher(input);
        final StringBuffer sb = new StringBuffer();
        final Pattern attrPattern = Pattern.compile("\\s[^ ]+=\"([^\"]*)\"");

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

				"application/fits",
				"application/font-tdpfr",
				"application/h224",
				"application/http",
				"application/hyperstudio",
				"application/ibe-key-request+xml",
				"application/ibe-pkg-reply+xml",
				"application/ibe-pp-data",
				"application/iges",
				"application/illustrator",
				"application/im-iscomposing+xml",
				"application/index",
				"application/index.cmd",

        final File targetFile = ResourceUtil.getResourceAsFile("test");
        String path = targetFile.getAbsolutePath();
        if (!path.startsWith("/")) {
            path = "/" + path.replace('\\', '/');
        }
        final String url = "file:" + path;

        final int maxCount = 3;
        final int numOfThread = 2;

        final File file = File.createTempFile("crawler-", "");

            responseData.setContentLength(0);
            ftpClientQueue.offer(client);
        }
    }

    /**
     * Validates the FTP request by checking the reply code from the FTP server.
     * If the reply code indicates an error, a CrawlingAccessException is thrown.
     *
     * @param client the FTPClient instance to validate
     * @throws CrawlingAccessException if the FTP request failed
     */

            final String text = source.get(FieldNames.TEXT).toString();
            if (skipDuplicateWords) {
                final String duplicateCheckStr = text.replace(" ", "");
                if (words.stream().map(word -> word.replace(" ", "")).anyMatch(word -> word.equals(duplicateCheckStr))) {
                    // skip duplicate word.
                    continue;
                }
            }

     * @param credentials The credentials.
     * @param value The value to replace.
     * @return The value with credentials replaced.
     */
    protected String replaceCredentials(final Credentials credentials, final String value) {
        if (StringUtil.isNotBlank(value)) {
            return value.replace(USERNAME, credentials.getUserPrincipal().getName()).replace(PASSWORD, credentials.getPassword());
        }

				<artifactId>maven-surefire-plugin</artifactId>
			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-deploy-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-javadoc-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-jar-plugin</artifactId>
				<configuration>
					<archive>

				<artifactId>maven-surefire-plugin</artifactId>
			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-deploy-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-javadoc-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-jar-plugin</artifactId>
				<configuration>
					<archive>