import java.io.InputStream
import java.io.InputStreamReader
import java.io.Reader
import java.nio.charset.Charset
import okhttp3.internal.charsetOrUtf8
import okhttp3.internal.chooseCharset
import okhttp3.internal.closeQuietly
import okhttp3.internal.readBomAsCharset
import okio.Buffer
import okio.BufferedSource
import okio.ByteString
import okio.use

/**

import okhttp3.HttpUrl
import okhttp3.Protocol
import okhttp3.Request
import okhttp3.Response
import okhttp3.Route
import okhttp3.internal.USER_AGENT
import okhttp3.internal.canReuseConnectionFor
import okhttp3.internal.closeQuietly
import okhttp3.internal.concurrent.TaskRunner
import okhttp3.internal.concurrent.withLock
import okhttp3.internal.connection.RoutePlanner.Plan
import okhttp3.internal.platform.Platform
import okhttp3.internal.toHostHeader

                responseData.setContentLength(0);
            }
        } catch (final CrawlerSystemException e) {
            CloseableUtil.closeQuietly(responseData);
            throw e;
        } catch (final SmbException e) {
            CloseableUtil.closeQuietly(responseData);
            throw new CrawlingAccessException("Could not access " + uri, e);
        }

        return responseData;
    }

    public void test_getText() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/msoffice/test.doc");
        final String content = msWordExtractor.getText(in, null).getContent();
        CloseableUtil.closeQuietly(in);
        logger.info(content);
        assertTrue(content.contains("テスト"));
    }

    public void test_getText_null() {
        try {
            msWordExtractor.getText(null, null);
            fail();

                        CloseableUtil.closeQuietly(p.getInputStream());
                    } catch (final Exception e) {
                        logger.warn("Could not close a process input stream.", e);
                    }
                });
                CommonPoolUtil.execute(() -> {
                    try {
                        CloseableUtil.closeQuietly(p.getErrorStream());

                                logger.debug("Exception in an internal extractor.", e);
                            }
                        } finally {
                            CloseableUtil.closeQuietly(is);
                        }
                    }
                }
            }
        } catch (final MaxLengthExceededException e) {
            throw e;
        } catch (final Exception e) {

                }
            }
        } catch (final CrawlerSystemException e) {
            CloseableUtil.closeQuietly(responseData);
            throw e;
        } catch (final Exception e) {
            CloseableUtil.closeQuietly(responseData);
            throw new CrawlingAccessException("Could not access " + uri, e);
        }
        return responseData;
    }

    /**

    public void test_getText() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/tar/test.tar");
        final String content = tarExtractor.getText(in, null).getContent();
        CloseableUtil.closeQuietly(in);
        logger.info(content);
        assertTrue(content.contains("テスト"));
        assertTrue(content.contains("テキスト"));
    }

    public void test_getText_maxSize() throws IOException {

                        try {
                            addSitemapsFromRobotsTxt(urlQueue);

                            if (responseData != null) {
                                CloseableUtil.closeQuietly(responseData);
                            }
                            if (crawlerContext.intervalController != null) {

import javax.net.ssl.SSLPeerUnverifiedException
import okhttp3.Interceptor
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.internal.canReuseConnectionFor
import okhttp3.internal.closeQuietly
import okhttp3.internal.connection.Exchange
import okhttp3.internal.connection.RealCall
import okhttp3.internal.http2.ConnectionShutdownException
import okhttp3.internal.stripBody
import okhttp3.internal.withSuppressed