View Javadoc

    1   // ========================================================================
    2   // Copyright 2006-2007 Mort Bay Consulting Pty. Ltd.
    3   // ------------------------------------------------------------------------
    4   // Licensed under the Apache License, Version 2.0 (the "License");
    5   // you may not use this file except in compliance with the License.
    6   // You may obtain a copy of the License at
    7   // http://www.apache.org/licenses/LICENSE-2.0
    8   // Unless required by applicable law or agreed to in writing, software
    9   // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  // ========================================================================
    14
    15  package org.mortbay.jetty.client;
    16
    17  import java.io.IOException;
    18  import java.io.InputStream;
    19  import java.net.UnknownHostException;
    20  import java.security.KeyStore;
    21  import java.security.SecureRandom;
    22  import java.util.Enumeration;
    23  import java.util.HashMap;
    24  import java.util.LinkedList;
    25  import java.util.Map;
    26  import java.util.Set;
    27
    28  import javax.net.ssl.HostnameVerifier;
    29  import javax.net.ssl.KeyManager;
    30  import javax.net.ssl.KeyManagerFactory;
    31  import javax.net.ssl.SSLContext;
    32  import javax.net.ssl.SSLSession;
    33  import javax.net.ssl.TrustManager;
    34  import javax.net.ssl.TrustManagerFactory;
    35  import javax.net.ssl.X509TrustManager;
    36
    37  import org.mortbay.component.LifeCycle;
    38  import org.mortbay.io.Buffer;
    39  import org.mortbay.io.ByteArrayBuffer;
    40  import org.mortbay.io.nio.DirectNIOBuffer;
    41  import org.mortbay.io.nio.IndirectNIOBuffer;
    42  import org.mortbay.jetty.AbstractBuffers;
    43  import org.mortbay.jetty.HttpSchemes;
    44  import org.mortbay.jetty.client.security.Authorization;
    45  import org.mortbay.jetty.client.security.RealmResolver;
    46  import org.mortbay.log.Log;
    47  import org.mortbay.resource.Resource;
    48  import org.mortbay.thread.QueuedThreadPool;
    49  import org.mortbay.thread.ThreadPool;
    50  import org.mortbay.thread.Timeout;
    51  import org.mortbay.util.Attributes;
    52  import org.mortbay.util.AttributesMap;
    53
    54  /**
    55   * Http Client.
    56   * <p/>
    57   * HttpClient is the main active component of the client API implementation.
    58   * It is the opposite of the Connectors in standard Jetty, in that it listens
    59   * for responses rather than requests.   Just like the connectors, there is a
    60   * blocking socket version and a non-blocking NIO version (implemented as nested classes
    61   * selected by {@link #setConnectorType(int)}).
    62   * <p/>
    63   * The an instance of {@link HttpExchange} is passed to the {@link #send(HttpExchange)} method
    64   * to send a request.  The exchange contains both the headers and content (source) of the request
    65   * plus the callbacks to handle responses.   A HttpClient can have many exchanges outstanding
    66   * and they may be queued on the {@link HttpDestination} waiting for a {@link HttpConnection},
    67   * queued in the {@link HttpConnection} waiting to be transmitted or pipelined on the actual
    68   * TCP/IP connection waiting for a response.
    69   * <p/>
    70   * The {@link HttpDestination} class is an aggregation of {@link HttpConnection}s for the
    71   * same host, port and protocol.   A destination may limit the number of connections
    72   * open and they provide a pool of open connections that may be reused.   Connections may also
    73   * be allocated from a destination, so that multiple request sources are not multiplexed
    74   * over the same connection.
    75   *
    76   * @see {@link HttpExchange}
    77   * @see {@link HttpDestination}
    78   * @author Greg Wilkins
    79   * @author Matthew Purland
    80   * @author Guillaume Nodet
    81   */
    82  public class HttpClient extends AbstractBuffers implements Attributes
    83  {
    84      public static final int CONNECTOR_SOCKET=0;
    85      public static final int CONNECTOR_SELECT_CHANNEL=2;
    86
    87      private int _connectorType=CONNECTOR_SELECT_CHANNEL;
    88      private boolean _useDirectBuffers=true;
    89      private int _maxConnectionsPerAddress=32;
    90      private Map<Address, HttpDestination> _destinations = new HashMap<Address, HttpDestination>();
    91      ThreadPool _threadPool;
    92      Connector _connector;
    93      private long _idleTimeout=20000;
    94      private long _timeout=320000;
    95      private int _soTimeout = 10000;
    96      private Timeout _timeoutQ = new Timeout();
    97      private Timeout _idleTimeoutQ = new Timeout();
    98      private Address _proxy;
    99      private Authorization _proxyAuthentication;
    100     private Set<String> _noProxy;
    101     private int _maxRetries = 3;
    102     private LinkedList<String> _registeredListeners;
    103
    104     // TODO clean up and add getters/setters to some of this maybe
    105     private String _keyStoreLocation;
    106     private String _keyStoreType="JKS";
    107     private String _keyStorePassword;
    108     private String _keyManagerAlgorithm = "SunX509";
    109     private String _keyManagerPassword;
    110     private String _trustStoreLocation;
    111     private String _trustStoreType="JKS";
    112     private String _trustStorePassword;
    113     private String _trustManagerAlgorithm = "SunX509";
    114
    115     private SSLContext _sslContext;
    116
    117     private String _protocol="TLS";
    118     private String _provider;
    119     private String _secureRandomAlgorithm;
    120
    121     private RealmResolver _realmResolver;
    122
    123     private AttributesMap _attributes=new AttributesMap();
    124
    125     /* ------------------------------------------------------------------------------- */
    126     public void dump() throws IOException
    127     {
    128         for (Map.Entry<Address, HttpDestination> entry : _destinations.entrySet())
    129         {
    130             System.err.println("\n"+entry.getKey()+":");
    131             entry.getValue().dump();
    132         }
    133     }
    134
    135     /* ------------------------------------------------------------------------------- */
    136     public void send(HttpExchange exchange) throws IOException
    137     {
    138         if (!isStarted())
    139             throw new IllegalStateException("!started");
    140         boolean ssl=HttpSchemes.HTTPS_BUFFER.equalsIgnoreCase(exchange.getScheme());
    141         exchange.setStatus(HttpExchange.STATUS_WAITING_FOR_CONNECTION);
    142         HttpDestination destination=getDestination(exchange.getAddress(),ssl);
    143         destination.send(exchange);
    144     }
    145
    146     /* ------------------------------------------------------------ */
    147     /**
    148      * @return the threadPool
    149      */
    150     public ThreadPool getThreadPool()
    151     {
    152         return _threadPool;
    153     }
    154
    155     /* ------------------------------------------------------------ */
    156     /**
    157      * @param threadPool the threadPool to set
    158      */
    159     public void setThreadPool(ThreadPool threadPool)
    160     {
    161         _threadPool=threadPool;
    162     }
    163
    164
    165     /* ------------------------------------------------------------ */
    166     /**
    167      * @param name
    168      * @return Attribute associated with client
    169      */
    170     public Object getAttribute(String name)
    171     {
    172         return _attributes.getAttribute(name);
    173     }
    174
    175     /* ------------------------------------------------------------ */
    176     /**
    177      * @return names of attributes associated with client
    178      */
    179     public Enumeration getAttributeNames()
    180     {
    181         return _attributes.getAttributeNames();
    182     }
    183
    184     /* ------------------------------------------------------------ */
    185     /**
    186      * @param name
    187      */
    188     public void removeAttribute(String name)
    189     {
    190         _attributes.removeAttribute(name);
    191     }
    192
    193     /* ------------------------------------------------------------ */
    194     /**
    195      * Set an attribute on the HttpClient.
    196      * Attributes are not used by the client, but are provided for
    197      * so that users of a shared HttpClient may share other structures.
    198      * @param name
    199      * @param attribute
    200      */
    201     public void setAttribute(String name, Object attribute)
    202     {
    203         _attributes.setAttribute(name,attribute);
    204     }
    205
    206     /* ------------------------------------------------------------ */
    207     /**
    208      * @param name
    209      * @return
    210      */
    211     public void clearAttributes()
    212     {
    213         _attributes.clearAttributes();
    214     }
    215
    216     /* ------------------------------------------------------------------------------- */
    217     public HttpDestination getDestination(Address remote, boolean ssl) throws UnknownHostException, IOException
    218     {
    219         if (remote==null)
    220             throw new UnknownHostException("Remote socket address cannot be null.");
    221
    222         synchronized (_destinations)
    223         {
    224             HttpDestination destination=_destinations.get(remote);
    225             if (destination==null)
    226             {
    227                 destination=new HttpDestination(this,remote,ssl,_maxConnectionsPerAddress);
    228                 if (_proxy != null && (_noProxy == null || !_noProxy.contains(remote.getHost())))
    229                 {
    230                     destination.setProxy(_proxy);
    231                     if (_proxyAuthentication!=null)
    232                         destination.setProxyAuthentication(_proxyAuthentication);
    233                 }
    234                 _destinations.put(remote,destination);
    235             }
    236             return destination;
    237         }
    238     }
    239
    240     /* ------------------------------------------------------------ */
    241     public void schedule(Timeout.Task task)
    242     {
    243         _timeoutQ.schedule(task);
    244     }
    245
    246     /* ------------------------------------------------------------ */
    247     public void schedule(Timeout.Task task, long timeout)
    248     {
    249         _timeoutQ.schedule(task, timeout);
    250     }
    251
    252     /* ------------------------------------------------------------ */
    253     public void scheduleIdle(Timeout.Task task)
    254     {
    255         _idleTimeoutQ.schedule(task);
    256     }
    257
    258     /* ------------------------------------------------------------ */
    259     public void cancel(Timeout.Task task)
    260     {
    261         task.cancel();
    262     }
    263
    264     /* ------------------------------------------------------------ */
    265     /**
    266      * Get whether the connector can use direct NIO buffers.
    267      */
    268     public boolean getUseDirectBuffers()
    269     {
    270         return _useDirectBuffers;
    271     }
    272
    273     /* ------------------------------------------------------------ */
    274     public void setRealmResolver( RealmResolver resolver )
    275     {
    276         _realmResolver = resolver;
    277     }
    278
    279     /* ------------------------------------------------------------ */
    280     /**
    281      * returns the SecurityRealmResolver registered with the HttpClient or null
    282      *
    283      * @return
    284      */
    285     public RealmResolver getRealmResolver()
    286     {
    287         return _realmResolver;
    288     }
    289
    290     /* ------------------------------------------------------------ */
    291     public boolean hasRealms()
    292     {
    293         return _realmResolver==null?false:true;
    294     }
    295
    296
    297     /**
    298      * Registers a listener that can listen to the stream of execution between the client and the
    299      * server and influence events.  Sequential calls to the method wrapper sequentially wrap the preceeding
    300      * listener in a delegation model.
    301      * <p/>
    302      * NOTE: the SecurityListener is a special listener which doesn't need to be added via this
    303      * mechanic, if you register security realms then it will automatically be added as the top listener of the
    304      * delegation stack.
    305      *
    306      * @param listenerClass
    307      */
    308     public void registerListener( String listenerClass )
    309     {
    310         if ( _registeredListeners == null )
    311         {
    312             _registeredListeners = new LinkedList<String>();
    313         }
    314         _registeredListeners.add( listenerClass );
    315     }
    316
    317     public LinkedList<String> getRegisteredListeners()
    318     {
    319         return _registeredListeners;
    320     }
    321
    322
    323     /* ------------------------------------------------------------ */
    324     /**
    325      * Set to use NIO direct buffers.
    326      *
    327      * @param direct
    328      *            If True (the default), the connector can use NIO direct
    329      *            buffers. Some JVMs have memory management issues (bugs) with
    330      *            direct buffers.
    331      */
    332     public void setUseDirectBuffers(boolean direct)
    333     {
    334         _useDirectBuffers=direct;
    335     }
    336
    337     /* ------------------------------------------------------------ */
    338     /**
    339      * Get the type of connector (socket, blocking or select) in use.
    340      */
    341     public int getConnectorType()
    342     {
    343         return _connectorType;
    344     }
    345
    346     /* ------------------------------------------------------------ */
    347     public void setConnectorType(int connectorType)
    348     {
    349         this._connectorType=connectorType;
    350     }
    351
    352     /* ------------------------------------------------------------ */
    353     /**
    354      * Create a new NIO buffer. If using direct buffers, it will create a direct
    355      * NIO buffer, other than an indirect buffer.
    356      */
    357     @Override
    358     protected Buffer newBuffer(int size)
    359     {
    360         if (_connectorType!=CONNECTOR_SOCKET)
    361         {
    362             Buffer buf=null;
    363             if (size==getHeaderBufferSize())
    364                 buf=new IndirectNIOBuffer(size);
    365             else if (_useDirectBuffers)
    366                 buf=new DirectNIOBuffer(size);
    367             else
    368                 buf=new IndirectNIOBuffer(size);
    369             return buf;
    370         }
    371         else
    372         {
    373             return new ByteArrayBuffer(size);
    374         }
    375     }
    376
    377     /* ------------------------------------------------------------ */
    378     public int getMaxConnectionsPerAddress()
    379     {
    380         return _maxConnectionsPerAddress;
    381     }
    382
    383     /* ------------------------------------------------------------ */
    384     public void setMaxConnectionsPerAddress(int maxConnectionsPerAddress)
    385     {
    386         _maxConnectionsPerAddress=maxConnectionsPerAddress;
    387     }
    388
    389     /* ------------------------------------------------------------ */
    390     protected void doStart() throws Exception
    391     {
    392         super.doStart();
    393
    394         _timeoutQ.setDuration(_timeout);
    395         _timeoutQ.setNow();
    396         _idleTimeoutQ.setDuration(_idleTimeout);
    397         _idleTimeoutQ.setNow();
    398
    399         if(_threadPool==null)
    400         {
    401             QueuedThreadPool pool = new QueuedThreadPool();
    402             pool.setMaxThreads(16);
    403             pool.setDaemon(true);
    404             pool.setName("HttpClient");
    405             _threadPool=pool;
    406         }
    407
    408         if (_threadPool instanceof LifeCycle)
    409         {
    410             ((LifeCycle)_threadPool).start();
    411         }
    412
    413
    414         if (_connectorType==CONNECTOR_SELECT_CHANNEL)
    415         {
    416
    417             _connector=new SelectConnector(this);
    418         }
    419         else
    420         {
    421             _connector=new SocketConnector(this);
    422         }
    423         _connector.start();
    424
    425         _threadPool.dispatch(new Runnable()
    426         {
    427             public void run()
    428             {
    429                 while (isRunning())
    430                 {
    431                     _timeoutQ.tick(System.currentTimeMillis());
    432                     _idleTimeoutQ.tick(_timeoutQ.getNow());
    433                     try
    434                     {
    435                         Thread.sleep(200);
    436                     }
    437                     catch (InterruptedException e)
    438                     {
    439                         Log.ignore(e);
    440                     }
    441                 }
    442             }
    443         });
    444
    445     }
    446
    447     /* ------------------------------------------------------------ */
    448     long getNow()
    449     {
    450         return _timeoutQ.getNow();
    451     }
    452
    453     /* ------------------------------------------------------------ */
    454     protected void doStop() throws Exception
    455     {
    456         _connector.stop();
    457         _connector=null;
    458         if (_threadPool instanceof LifeCycle)
    459         {
    460             ((LifeCycle)_threadPool).stop();
    461         }
    462         for (HttpDestination destination : _destinations.values())
    463         {
    464             destination.close();
    465         }
    466
    467         _timeoutQ.cancelAll();
    468         _idleTimeoutQ.cancelAll();
    469         super.doStop();
    470     }
    471
    472     /* ------------------------------------------------------------ */
    473     interface Connector extends LifeCycle
    474     {
    475         public void startConnection(HttpDestination destination) throws IOException;
    476
    477     }
    478
    479     /**
    480      * if a keystore location has been provided then client will attempt to use it as the keystore,
    481      * otherwise we simply ignore certificates and run with a loose ssl context.
    482      *
    483      * @return
    484      * @throws IOException
    485      */
    486     protected SSLContext getSSLContext() throws IOException
    487     {
    488    	if (_sslContext == null)
    489     	{
    490             if (_keyStoreLocation == null)
    491             {
    492                 _sslContext = getLooseSSLContext();
    493             }
    494             else
    495             {
    496                 _sslContext = getStrictSSLContext();
    497             }
    498         }
    499     	return _sslContext;
    500     }
    501
    502     protected SSLContext getStrictSSLContext() throws IOException
    503     {
    504
    505         try
    506         {
    507             if (_trustStoreLocation==null)
    508             {
    509                 _trustStoreLocation=_keyStoreLocation;
    510                 _trustStoreType=_keyStoreType;
    511             }
    512
    513             KeyManager[] keyManagers=null;
    514
    515             InputStream keystoreInputStream = null;
    516             keystoreInputStream= Resource.newResource(_keyStoreLocation).getInputStream();
    517
    518             KeyStore keyStore=KeyStore.getInstance(_keyStoreType);
    519             keyStore.load(keystoreInputStream,_keyStorePassword==null?null:_keyStorePassword.toString().toCharArray());
    520             keystoreInputStream.close();
    521
    522             KeyManagerFactory keyManagerFactory=KeyManagerFactory.getInstance(_keyManagerAlgorithm);
    523             keyManagerFactory.init(keyStore,_keyManagerPassword==null?null:_keyManagerPassword.toString().toCharArray());
    524             keyManagers=keyManagerFactory.getKeyManagers();
    525
    526             TrustManager[] trustManagers=null;
    527
    528             InputStream truststoreInputStream = null;
    529             truststoreInputStream = Resource.newResource(_trustStoreLocation).getInputStream();
    530
    531             KeyStore trustStore=KeyStore.getInstance(_trustStoreType);
    532             trustStore.load(truststoreInputStream,_trustStorePassword==null?null:_trustStorePassword.toString().toCharArray());
    533             truststoreInputStream.close();
    534
    535             TrustManagerFactory trustManagerFactory=TrustManagerFactory.getInstance(_trustManagerAlgorithm);
    536             trustManagerFactory.init(trustStore);
    537             trustManagers=trustManagerFactory.getTrustManagers();
    538
    539             SecureRandom secureRandom=_secureRandomAlgorithm==null?null:SecureRandom.getInstance(_secureRandomAlgorithm);
    540             SSLContext context=_provider==null?SSLContext.getInstance(_protocol):SSLContext.getInstance(_protocol,_provider);
    541             context.init(keyManagers,trustManagers,secureRandom);
    542             return context;
    543         }
    544         catch ( Exception e )
    545         {
    546             Log.debug(e);
    547             throw new IOException( "error generating ssl context for " + _keyStoreLocation  + " " + e.getMessage() );
    548         }
    549     }
    550
    551     protected SSLContext getLooseSSLContext() throws IOException
    552     {
    553
    554         // Create a trust manager that does not validate certificate
    555         // chains
    556         TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager()
    557         {
    558             public java.security.cert.X509Certificate[] getAcceptedIssuers()
    559             {
    560                 return null;
    561             }
    562
    563             public void checkClientTrusted( java.security.cert.X509Certificate[] certs, String authType )
    564             {
    565             }
    566
    567             public void checkServerTrusted( java.security.cert.X509Certificate[] certs, String authType )
    568             {
    569             }
    570         } };
    571
    572         HostnameVerifier hostnameVerifier = new HostnameVerifier()
    573         {
    574             public boolean verify( String urlHostName, SSLSession session )
    575             {
    576                 Log.warn( "Warning: URL Host: " + urlHostName + " vs." + session.getPeerHost() );
    577                 return true;
    578             }
    579         };
    580
    581         // Install the all-trusting trust manager
    582         try
    583         {
    584             // TODO real trust manager
    585             SSLContext sslContext = SSLContext.getInstance( "SSL" );
    586             sslContext.init( null, trustAllCerts, new java.security.SecureRandom() );
    587             return sslContext;
    588         }
    589         catch ( Exception e )
    590         {
    591             Log.debug(e);
    592             throw new IOException( "issue ignoring certs" );
    593         }
    594     }
    595
    596     /* ------------------------------------------------------------ */
    597     /**
    598      * @return the period in milliseconds a {@link HttpConnection} can be idle for before it is closed.
    599      */
    600     public long getIdleTimeout()
    601     {
    602         return _idleTimeout;
    603     }
    604
    605     /* ------------------------------------------------------------ */
    606     /**
    607      * @param ms the period in milliseconds a {@link HttpConnection} can be idle for before it is closed.
    608      */
    609     public void setIdleTimeout(long ms)
    610     {
    611         _idleTimeout=ms;
    612     }
    613
    614     /* ------------------------------------------------------------ */
    615     public int getSoTimeout()
    616     {
    617         return _soTimeout;
    618     }
    619
    620     /* ------------------------------------------------------------ */
    621     public void setSoTimeout(int so)
    622     {
    623         _soTimeout = so;
    624     }
    625
    626     /* ------------------------------------------------------------ */
    627     /**
    628      * @return the period in ms that an exchange will wait for a response from the server.
    629      */
    630     public long getTimeout()
    631     {
    632         return _timeout;
    633     }
    634
    635     /* ------------------------------------------------------------ */
    636     /**
    637      * @param ms the period in ms that an exchange will wait for a response from the server.
    638      */
    639     public void setTimeout(long ms)
    640     {
    641         _timeout=ms;
    642     }
    643
    644     /* ------------------------------------------------------------ */
    645     public Address getProxy()
    646     {
    647         return _proxy;
    648     }
    649
    650     /* ------------------------------------------------------------ */
    651     public void setProxy(Address proxy)
    652     {
    653         this._proxy = proxy;
    654     }
    655
    656     /* ------------------------------------------------------------ */
    657     public Authorization getProxyAuthentication()
    658     {
    659         return _proxyAuthentication;
    660     }
    661
    662     /* ------------------------------------------------------------ */
    663     public void setProxyAuthentication(Authorization authentication)
    664     {
    665         _proxyAuthentication = authentication;
    666     }
    667
    668     /* ------------------------------------------------------------ */
    669     public boolean isProxied()
    670     {
    671         return this._proxy!=null;
    672     }
    673
    674     /* ------------------------------------------------------------ */
    675     public Set<String> getNoProxy()
    676     {
    677         return _noProxy;
    678     }
    679
    680     /* ------------------------------------------------------------ */
    681     public void setNoProxy(Set<String> noProxyAddresses)
    682     {
    683         _noProxy = noProxyAddresses;
    684     }
    685
    686     /* ------------------------------------------------------------ */
    687     public int maxRetries()
    688     {
    689         return _maxRetries;
    690     }
    691
    692      /* ------------------------------------------------------------ */
    693     public void setMaxRetries( int retries )
    694     {
    695         _maxRetries = retries;
    696     }
    697
    698     /* ------------------------------------------------------------ */
    699     public String getTrustStoreLocation()
    700     {
    701         return _trustStoreLocation;
    702     }
    703
    704     /* ------------------------------------------------------------ */
    705     public void setTrustStoreLocation(String trustStoreLocation)
    706     {
    707         this._trustStoreLocation = trustStoreLocation;
    708     }
    709
    710     /* ------------------------------------------------------------ */
    711     public String getKeyStoreLocation()
    712     {
    713         return _keyStoreLocation;
    714     }
    715
    716     /* ------------------------------------------------------------ */
    717     public void setKeyStoreLocation(String keyStoreLocation)
    718     {
    719         this._keyStoreLocation = keyStoreLocation;
    720     }
    721
    722     /* ------------------------------------------------------------ */
    723     public void setKeyStorePassword(String _keyStorePassword)
    724     {
    725         this._keyStorePassword = _keyStorePassword;
    726     }
    727
    728     /* ------------------------------------------------------------ */
    729     public void setKeyManagerPassword(String _keyManagerPassword)
    730     {
    731         this._keyManagerPassword = _keyManagerPassword;
    732     }
    733
    734     /* ------------------------------------------------------------ */
    735     public void setTrustStorePassword(String _trustStorePassword)
    736     {
    737         this._trustStorePassword = _trustStorePassword;
    738     }
    739 }