- **Flexible Configuration**: XML-based dependency injection with LastaFlute DI
- **Extensible Architecture**: Plugin system for custom extractors, transformers, and clients
- **Rate Limiting**: Politeness policies and interval controllers
- **URL Filtering**: Regex-based inclusion/exclusion patterns
- **Data Persistence**: Multiple backend options including OpenSearch integration

## Technology Stack

    }

    /**
     * Gets the last modified date of the resource.
     *
     * @return the last modified date, or null if not available
     */
    public Date getLastModified() {
        return lastModified;
    }

    /**
     * Sets the last modified date of the resource.
     *
     * @param lastModified the last modified date to set
     */

    }

    /**
     * Gets the received date from a message by parsing the received headers.
     *
     * @param message the message to get the received date from
     * @return the received date or null if not found
     * @throws MessagingException if message access fails
     */
    protected static Date getReceivedDate(final Message message) throws MessagingException {
        final Date today = new Date();

            server.stop();
        }
    }

    public void test_parseLastModified() {
        final String value = "Mon, 01 Jun 2009 21:02:45 GMT";
        final Date date = httpClient.parseLastModifiedDate(value);
        assertNotNull(date);
    }

    public void test_processRobotsTxt() {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

    }

    /**
     * Extracts a Date object from the source map using the specified field name.
     * Handles various date formats including Date objects, timestamp numbers, and ISO date strings.
     *
     * @param sourceMap The source map containing the date field.
     * @param name The name of the date field.
     * @return The Date object, or null if the field doesn't contain a valid date.
     */

        }

        /**
         * Gets the date of the storage object.
         * This method returns the same value as getLastModified().
         *
         * @return The date in milliseconds since epoch
         */
        @Override
        public long getDate() {
            return getLastModified();
        }

        /**
         * Returns the last modified date of the storage object.

                responseData.setHttpStatusCode(Constants.OK_STATUS_CODE);
                responseData.setCharSet(getCharSet(file));
                responseData.setLastModified(new Date(file.lastModified()));
                responseData.addMetaData(SMB_CREATE_TIME, new Date(file.createTime()));
                try {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Parsing SmbFile Owner: {}", filePath);

import java.nio.file.attribute.AclFileAttributeView;
import java.nio.file.attribute.FileOwnerAttributeView;
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.UserPrincipal;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

                responseData.setHttpStatusCode(Constants.OK_STATUS_CODE);
                responseData.setCharSet(getCharSet(file));
                responseData.setLastModified(new Date(file.lastModified()));
                responseData.addMetaData(SMB_CREATE_TIME, new Date(file.createTime()));
                try {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Parsing SmbFile Owner: {}", filePath);

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client.ftp;

import java.io.File;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.ftpserver.FtpServer;
import org.apache.ftpserver.FtpServerFactory;