dfos = out;
            CopyUtil.copy(in, out);
            out.flush();

            Extractor extractor = StringUtil.isBlank(mimeType) ? null : extractorFactory.getExtractor(mimeType);
            if (extractor == null) {
                final String detectedMimeType = getMimeType(out);

         * @throws CrawlingAccessException If the URL is invalid or malformed
         */
        public FtpInfo(final String s, final String c) {
            if (StringUtil.isBlank(s)) {
                throw new CrawlingAccessException("uri is blank.");
            }

            try {
                uri = new URI(normalize(s).replace("%", "%25").replace(" ", "%20"));
            } catch (final URISyntaxException e) {

            return name.substring(pos + 1);
        }
        return name;
    }

    private void executeCommand(final File inputFile, final File outputFile) {

        if (StringUtil.isBlank(command)) {
            throw new CrawlerSystemException("command is empty.");
        }

        final Map<String, String> params = new HashMap<>();
        params.put("$INPUT_FILE", inputFile.getAbsolutePath());

        final String loginUrl = replaceCredentials(credentials, originalLoginUrl);
        final String loginReqParams = replaceCredentials(credentials, getParameter(LOGIN_PARAMETERS));

        if (StringUtil.isBlank(loginUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("No login.url");
            }
            return;
        }

        final HttpUriRequest httpRequest;

     * @param urlQueue The URL queue entry to validate.
     * @return true if valid, false otherwise.
     */
    protected boolean isValid(final UrlQueue<?> urlQueue) {
        if (urlQueue == null || StringUtil.isBlank(urlQueue.getUrl())
                || crawlerContext.getMaxDepth() >= 0 && urlQueue.getDepth() > crawlerContext.getMaxDepth()) {
            return false;
        }

        // url filter

     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {
            throw new CrawlerSystemException("url is null or empty.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {
            // not support robots.txt