# MIME types for HTML document cache.
crawler.document.cache.html.mimetypes=text/html
# Extension-to-MIME-type override mappings for MIME type detection (one per line: .ext=mime/type).
crawler.document.mimetype.extension.overrides=

# indexer

# Whether to enable thread dump for the indexer.

// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package subnet

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"time"

	xhttp "github.com/minio/minio/internal/http"
)

const (
	respBodyLimit = 1 << 20 // 1 MiB

	// LoggerWebhookName - subnet logger webhook target
	LoggerWebhookName = "subnet"
)

     */
    @Override
    public Logger getLogger() {
        return logger;
    }

    /**
     * Gets the appropriate extractor for the given response data.
     * Selects an extractor based on the MIME type or falls back to the Tika extractor.
     *
     * @param responseData the response data containing the document to extract
     * @return the extractor instance for processing the document

     */
    public static final String METHOD = "method";

    /**
     * Field name for MIME type.
     */
    public static final String MIME_TYPE = "mimeType";

    /**
     * Field name for creation time.
     */
    public static final String CREATE_TIME = "createTime";

    /**
     * Field name for execution time.
     */
    public static final String EXECUTION_TIME = "executionTime";

    /**

    private static final Logger logger = LogManager.getLogger(AbstractDataStore.class);

    /**
     * The script type.
     */
    protected static final String SCRIPT_TYPE = "script_type";

    /**
     * The mime type.
     */
    public String mimeType = "application/datastore";

    /**
     * The flag to check if the data store is alive.
     * Volatile to ensure visibility across threads.
     */

## Content-Type

        this.commandDestroyTimeout = commandDestroyTimeout;
    }

    /**
     * Gets file extension from MIME type for creating temp files with proper extensions.
     * This helps ImageMagick correctly identify file formats.
     * @param mimeType The MIME type of the content.
     * @return The file extension including the dot (e.g., ".gif"), or empty string if unknown.
     */

  if [[ -z "${im_cmd}" ]] ; then
    echo "ImageMagick (convert or magick) does not exist."
    exit 1
  fi
  target_file=$(echo "$url" | sed -e "s#^file:/*#/#g")
  # Build format hint from MIME type for better ImageMagick compatibility
  format_hint=""
  case "${mimetype}" in
    "image/gif")  format_hint="gif:" ;;
    "image/tiff") format_hint="tiff:" ;;
    "image/png")  format_hint="png:" ;;

 */
public abstract class AbstractCrawlerClient implements CrawlerClient {

    private static final Logger logger = LogManager.getLogger(AbstractCrawlerClient.class);

    /** The MIME type for application/octet-stream. */
    protected static final String APPLICATION_OCTET_STREAM = "application/octet-stream";

    /** The property name for access timeout. */

            buf.append('\"').append(StringEscapeUtils.escapeJson(obj.toString())).append('\"');
        }
        return buf.toString();
    }

    /**
     * Sets the MIME type.
     * @param mimeType The MIME type.
     */
    public void setMimeType(final String mimeType) {
        this.mimeType = mimeType;
    }