* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.rule.impl;

import java.io.File;
import java.io.InputStream;
import java.util.regex.Pattern;

import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.io.ResourceUtil;

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml"/>
	<include path="crawler/client.xml"/>
	<include path="crawler/rule.xml"/>
	<include path="crawler/filter.xml"/>
	<include path="crawler/interval.xml"/>
	<include path="crawler/extractor.xml"/>
	<include path="crawler/mimetype.xml"/>
	<include path="crawler/encoding.xml"/>

netsh interface portproxy add v4tov4 listenaddress=$IP_ADDR listenport=80 connectaddress=169.254.169.254 connectport=80
# A local firewall rule for the container is added in

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
    <include path="crawler/container.xml"/>
    <include path="crawler/client.xml"/>
    <include path="crawler/rule.xml"/>
    <include path="crawler/filter.xml"/>
    <include path="crawler/interval.xml"/>
    <include path="crawler/extractor.xml"/>
    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>

                            artifactRules =
                                if (index == 0) {
                                    // The artifact rule report/css/*.css => performanceResultsDir is there to clean up the target directory.
                                    // If we don't clean that up there might be leftover json files from other report builds running on the same machine.

    # Additional setup is contained in ci/official/envs/rbe.
    CONTAINER_IP_ADDR=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tf)
    netsh advfirewall firewall add rule name="Allow Metadata Proxy" dir=in action=allow protocol=TCP localport=80 remoteip="$CONTAINER_IP_ADDR"
  fi

fi

    controller.setDefaultIntervalTime(1000);
});
```

### Sitemap Support

```java
// Enable sitemap processing
container.singleton("sitemapsRule", SitemapsRule.class, rule -> {
    rule.addRule("url", ".*sitemap.*");
});

// Add sitemap URL
crawler.addUrl("https://example.com/sitemap.xml");
```

## Data Access and Storage

### Accessing Crawled Data

```java

  netsh interface portproxy add v4tov4 listenaddress=$IP_ADDR listenport=80 connectaddress=169.254.169.254 connectport=80
  # A local firewall rule for the container is added in
  # ci/official/utilities/setup_docker.sh.
else
  # The volume mapping flag below shares the user's gcloud credentials, if any,
  # with the container, in case the user has credentials stored there.

sleep 10s

## Add warm tier
./mc ilm tier add minio sitea WARM-TIER --endpoint http://localhost:9004 --access-key minioadmin --secret-key minioadmin --bucket bucket

## Add ILM rules
./mc ilm add sitea/bucket --transition-days 0 --transition-tier WARM-TIER
./mc ilm rule list sitea/bucket

./mc cp README.md sitea/bucket/README.md

until $(./mc stat sitea/bucket/README.md --json | jq -r '.metadata."X-Amz-Storage-Class"' | grep -q WARM-TIER); do

    START_THREAD,
    /** Indicates the finish of a crawler thread. */
    FINISHED_THREAD,
    /** Indicates no response processor is available. */
    NO_RESPONSE_PROCESSOR,
    /** Indicates no rule is available for processing. */
    NO_RULE,
    /** Indicates a system error occurred. */
    SYSTEM_ERROR