* either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.extractor.impl;

import java.util.regex.Pattern;

/**
 * Extracts text content from XML documents.
 */
public class XmlExtractor extends AbstractXmlExtractor {

    /**
     * Creates a new XmlExtractor instance.
     */

 * represents a valid sitemap. It uses a SitemapsHelper to validate the response body as an InputStream.
 * The rule checks if the URL matches the defined regex pattern and then validates the content as a sitemap.
 * If any exception occurs during the sitemap validation, it logs the error and returns false.
 *
 */
public class SitemapsRule extends RegexRule {
    /**

      <match value="[0-9]{5,5}" type="regex" offset="0">
        <match value="45" type="string" offset="20">
          <!-- bibliographic -->
          <match value="[acdnp][acdefgijkmoprt][abcdims]" type="regex" offset="5"/>
          <!-- authority-->
          <match value="[acdnosx]z" type="regex" offset="5"/>
          <!-- holdings -->
          <match value="[cdn][uvxy]" type="regex" offset="5"/>
          <!-- classification -->

- **Extensible Architecture**: Plugin system for custom extractors, transformers, and clients
- **Rate Limiting**: Politeness policies and interval controllers
- **URL Filtering**: Regex-based inclusion/exclusion patterns
- **Data Persistence**: Multiple backend options including OpenSearch integration

## Technology Stack

- **Java**: 21+ (requires Java 21 or higher)
- **Build System**: Maven 3.x

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.service;

import java.util.List;
import java.util.regex.Pattern;

/**
 * Service interface for managing URL filters.
 * Provides methods to add and remove include/exclude URL filters,
 * as well as retrieve the patterns of these filters.
 */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.rule.impl;

import java.io.File;
import java.io.InputStream;
import java.util.regex.Pattern;

import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ResponseData;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.ByteOrderMark;
import org.apache.commons.io.input.BOMInputStream;
import org.apache.commons.text.translate.AggregateTranslator;

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.rule.impl;

import java.util.regex.Pattern;

import org.codelibs.fess.crawler.entity.ResponseData;
import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class RegexRuleTest extends PlainTestCase {

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.service.impl;

import java.util.List;
import java.util.regex.Pattern;

import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.UrlFilterService;

import jakarta.annotation.Resource;

/**

import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.xpath.XPathEvaluationResult;
import javax.xml.xpath.XPathException;
import javax.xml.xpath.XPathNodes;

import org.apache.logging.log4j.LogManager;