public void addMaxLength(final String mimeType, final long maxLength) {
        if (StringUtil.isBlank(mimeType)) {
            throw new CrawlerSystemException("MIME type is a blank.");
        }
        if (maxLength < 0) {
            throw new CrawlerSystemException("The value of maxLength is invalid.");
        }
        maxLengthMap.put(mimeType, maxLength);
    }

        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url)) {
            if (logger.isDebugEnabled()) {
                logger.debug("URL is a blank: {}", url);
            }
            return false;
        }

        // check it in queue
        for (final UrlQueue<Long> urlInQueue : urlQueueList) {
            if (url.equals(urlInQueue.getUrl())) {

        if (StringUtil.isBlank(endpoint)) {
            throw new CrawlingAccessException("endpoint is blank.");
        }
        builder.endpoint(endpoint);
        final String accessKey = getInitParameter("accessKey", null, String.class);
        if (StringUtil.isBlank(accessKey)) {
            throw new CrawlingAccessException("accessKey is blank.");
        }
        final String secretKey = getInitParameter("secretKey", null, String.class);

            final String region = System.getenv().get("STORAGE_SECRET_KEY");
            try {
                if (StringUtil.isBlank(endpoint)) {
                    throw new IOException("endpoint is blank.");
                }
                final Builder builder = MinioClient.builder().endpoint(endpoint);
                if (StringUtil.isNotBlank(accessKey) && StringUtil.isNotBlank(secretKey)) {

    public boolean visited(final EsUrlQueue urlQueue) {
        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url)) {
            if (logger.isDebugEnabled()) {
                logger.debug("URL is a blank: {}", url);
            }
            return false;
        }

        final String sessionId = urlQueue.getSessionId();
        if (super.exists(sessionId, url)) {
            return true;
        }

        final String fileName = artifact.getFileName();
        final String url = artifact.getUrl();
        if (StringUtil.isBlank(url)) {
            throw new PluginException("url is blank: " + artifact.getName());
        }
        if (url.startsWith("http:") || url.startsWith("https:")) {
            try (final CurlResponse response = createCurlRequest(url).execute()) {