<?xml version="1.0" encoding="UTF-8"?>
<metadata>
  <groupId>org.codelibs.fess</groupId>
  <artifactId>fess-crawler-webdriver</artifactId>
  <versioning>
    <latest>3.16.0</latest>
    <release>3.16.0</release>
    <versions>
      <version>1.0.3</version>
      <version>1.0.4</version>
      <version>1.0.5</version>
      <version>1.0.6</version>
      <version>1.0.7</version>
      <version>1.0.8</version>
      <version>1.0.9</version>

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

import java.io.IOException;

import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**

        scheduledJobBhv.insertOrUpdate(scheduledJob, op -> {
            op.setRefreshPolicy(Constants.TRUE);
        });
    }

    /**
     * Gets a list of crawler jobs.
     * @return A list of crawler jobs.
     */
    public List<ScheduledJob> getCrawlerJobList() {
        return scheduledJobBhv.selectList(cb -> {
            cb.query().setCrawler_Equal(Constants.T);

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client.http.conn;

import org.dbflute.utflute.core.PlainTestCase;

public class IdnDnsResolverTest extends PlainTestCase {

    public void test_toAscii() {
        IdnDnsResolver resolver = new IdnDnsResolver();

import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.serializer.DataSerializer;
import org.codelibs.fess.crawler.util.FieldConfigs;

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

/**
 * Constants used in the fess-crawler.
 * This class provides a collection of constant values for HTTP methods, status codes,

package callbacks

import (
	"reflect"

	"gorm.io/gorm"
)

func callMethod(db *gorm.DB, fc func(value interface{}, tx *gorm.DB) bool) {
	tx := db.Session(&gorm.Session{NewDB: true})
	if called := fc(db.Statement.ReflectValue.Interface(), tx); !called {
		switch db.Statement.ReflectValue.Kind() {
		case reflect.Slice, reflect.Array:
			db.Statement.CurDestIndex = 0
			for i := 0; i < db.Statement.ReflectValue.Len(); i++ {

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * RobotsTxtException is an exception class that represents an error related to robots.txt processing during web crawling.

        });

        try {
            crawlJob.execute();
            fail("Should throw JobProcessingException");
        } catch (JobProcessingException e) {
            assertTrue(e.getMessage().contains("10 crawler processes are running"));
            assertTrue(e.getMessage().contains("Max processes are 5"));
        }
    }

    // Test execute method with no max process limit

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler;

import java.util.List;
import java.util.regex.Pattern;

import org.codelibs.core.misc.Pair;
import org.codelibs.fess.unit.UnitFessTestCase;

public class FessCrawlerThreadTest extends UnitFessTestCase {