WebResource.java |
1 /** 2 * Distribution License: 3 * JSword is free software; you can redistribute it and/or modify it under 4 * the terms of the GNU Lesser General Public License, version 2.1 or later 5 * as published by the Free Software Foundation. This program is distributed 6 * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even 7 * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 8 * See the GNU Lesser General Public License for more details. 9 * 10 * The License is available on the internet at: 11 * http://www.gnu.org/copyleft/lgpl.html 12 * or by writing to: 13 * Free Software Foundation, Inc. 14 * 59 Temple Place - Suite 330 15 * Boston, MA 02111-1307, USA 16 * 17 * © CrossWire Bible Society, 2005 - 2016 18 * 19 */ 20 package org.crosswire.common.util; 21 22 import java.io.IOException; 23 import java.io.InputStream; 24 import java.io.OutputStream; 25 import java.net.URI; 26 import java.util.Date; 27 28 import org.apache.http.Header; 29 import org.apache.http.HttpEntity; 30 import org.apache.http.HttpHost; 31 import org.apache.http.HttpResponse; 32 import org.apache.http.HttpStatus; 33 import org.apache.http.StatusLine; 34 import org.apache.http.client.config.RequestConfig; 35 import org.apache.http.client.methods.HttpGet; 36 import org.apache.http.client.methods.HttpHead; 37 import org.apache.http.client.methods.HttpRequestBase; 38 import org.apache.http.client.utils.DateUtils; 39 import org.apache.http.impl.client.CloseableHttpClient; 40 import org.apache.http.impl.client.HttpClientBuilder; 41 import org.crosswire.common.progress.Progress; 42 import org.crosswire.jsword.JSMsg; 43 44 /** 45 * A WebResource is backed by an URL and potentially the proxy through which it 46 * need go. It can get basic information about the resource and it can get the 47 * resource. The requests are subject to a timeout, which can be set via the 48 * constructor or previously by a call to set the default timeout. The initial 49 * default timeout is 750 milliseconds. 50 * 51 * 52 * @see gnu.lgpl.License The GNU Lesser General Public License for details. 53 * @author DM Smith 54 */ 55 public class WebResource { 56 /** 57 * Construct a WebResource for the given URL, while timing out if too much 58 * time has passed. 59 * 60 * @param theURI 61 * the Resource to get via HTTP 62 */ 63 public WebResource(URI theURI) { 64 this(theURI, null, null, timeout); 65 } 66 67 /** 68 * Construct a WebResource for the given URL, while timing out if too much 69 * time has passed. 70 * 71 * @param theURI 72 * the Resource to get via HTTP 73 * @param theTimeout 74 * the length of time in milliseconds to allow a connection to 75 * respond before timing out 76 */ 77 public WebResource(URI theURI, int theTimeout) { 78 this(theURI, null, null, theTimeout); 79 } 80 81 /** 82 * Construct a WebResource for the given URL, going through the optional 83 * proxy and default port, while timing out if too much time has passed. 84 * 85 * @param theURI 86 * the Resource to get via HTTP 87 * @param theProxyHost 88 * the proxy host or null 89 */ 90 public WebResource(URI theURI, String theProxyHost) { 91 this(theURI, theProxyHost, null, timeout); 92 } 93 94 /** 95 * Construct a WebResource for the given URL, going through the optional 96 * proxy and default port, while timing out if too much time has passed. 97 * 98 * @param theURI 99 * the Resource to get via HTTP 100 * @param theProxyHost 101 * the proxy host or null 102 * @param theTimeout 103 * the length of time in milliseconds to allow a connection to 104 * respond before timing out 105 */ 106 public WebResource(URI theURI, String theProxyHost, int theTimeout) { 107 this(theURI, theProxyHost, null, theTimeout); 108 } 109 110 /** 111 * Construct a WebResource for the given URL, going through the optional 112 * proxy and port, while timing out if too much time has passed. 113 * 114 * @param theURI 115 * the Resource to get via HTTP 116 * @param theProxyHost 117 * the proxy host or null 118 * @param theProxyPort 119 * the proxy port or null, where null means use the standard port 120 */ 121 public WebResource(URI theURI, String theProxyHost, Integer theProxyPort) { 122 this(theURI, theProxyHost, theProxyPort, timeout); 123 } 124 125 /** 126 * Construct a WebResource for the given URL, going through the optional 127 * proxy and port, while timing out if too much time has passed. 128 * 129 * @param theURI 130 * the Resource to get via HTTP 131 * @param theProxyHost 132 * the proxy host or null 133 * @param theProxyPort 134 * the proxy port or null, where null means use the standard port 135 * @param theTimeout 136 * the length of time in milliseconds to allow a connection to 137 * respond before timing out 138 */ 139 public WebResource(URI theURI, String theProxyHost, Integer theProxyPort, int theTimeout) { 140 uri = theURI; 141 HttpHost proxy = null; 142 143 // Configure proxy info if necessary and defined 144 if (theProxyHost != null && theProxyHost.length() > 0) { 145 proxy = new HttpHost(theProxyHost, theProxyPort == null ? -1 : theProxyPort.intValue()); 146 } 147 148 final RequestConfig.Builder builder = RequestConfig.custom(); 149 builder.setConnectTimeout(theTimeout).setConnectionRequestTimeout(theTimeout).setSocketTimeout(theTimeout).setProxy(proxy); 150 client = HttpClientBuilder.create().setDefaultRequestConfig(builder.build()).build(); 151 } 152 153 /** 154 * When this WebResource is no longer needed it should be shutdown to return 155 * underlying resources back to the OS. 156 */ 157 public void shutdown() { 158 IOUtil.close(client); 159 } 160 161 /** 162 * @return the timeout in milliseconds 163 */ 164 public static int getTimeout() { 165 return timeout; 166 } 167 168 /** 169 * @param timeout 170 * the timeout to set in milliseconds 171 */ 172 public static void setTimeout(int timeout) { 173 WebResource.timeout = timeout; 174 } 175 176 /** 177 * Determine the size of this WebResource. 178 * <p> 179 * Note that the http client may read the entire file to determine this. 180 * </p> 181 * 182 * @return the size of the file 183 */ 184 public int getSize() { 185 HttpRequestBase method = new HttpHead(uri); 186 HttpResponse response = null; 187 try { 188 // Execute the method. 189 response = client.execute(method); 190 StatusLine statusLine = response.getStatusLine(); 191 if (statusLine.getStatusCode() == HttpStatus.SC_OK) { 192 return getHeaderAsInt(response, "Content-Length"); 193 } 194 String reason = response.getStatusLine().getReasonPhrase(); 195 // TRANSLATOR: Common error condition: {0} is a placeholder for the 196 // URL of what could not be found. 197 Reporter.informUser(this, JSMsg.gettext("Unable to find: {0}", reason + ':' + uri.getPath())); 198 } catch (IOException e) { 199 return 0; 200 } 201 return 0; 202 } 203 204 /** 205 * Determine the last modified date of this WebResource. 206 * <p> 207 * Note that the http client may read the entire file. 208 * </p> 209 * 210 * @return the last mod date of the file 211 */ 212 public long getLastModified() { 213 HttpRequestBase method = new HttpHead(uri); 214 HttpResponse response = null; 215 try { 216 // Execute the method. 217 response = client.execute(method); 218 StatusLine statusLine = response.getStatusLine(); 219 if (statusLine.getStatusCode() == HttpStatus.SC_OK) { 220 return getHeaderAsDate(response, "Last-Modified"); 221 } 222 String reason = response.getStatusLine().getReasonPhrase(); 223 // TRANSLATOR: Common error condition: {0} is a placeholder for the 224 // URL of what could not be found. 225 Reporter.informUser(this, JSMsg.gettext("Unable to find: {0}", reason + ':' + uri.getPath())); 226 } catch (IOException e) { 227 return new Date().getTime(); 228 } 229 return new Date().getTime(); 230 } 231 232 /** 233 * Copy this WebResource to the destination and report progress. 234 * 235 * @param dest 236 * the URI of the destination, typically a file:///. 237 * @param meter 238 * the job on which to report progress 239 * @throws LucidException when an error is encountered 240 */ 241 public void copy(URI dest, Progress meter) throws LucidException { 242 InputStream in = null; 243 OutputStream out = null; 244 HttpRequestBase method = new HttpGet(uri); 245 HttpResponse response = null; 246 HttpEntity entity = null; 247 try { 248 // Execute the method. 249 response = client.execute(method); 250 // Initialize the meter, if present 251 if (meter != null) { 252 // Find out how big it is 253 int size = getHeaderAsInt(response, "Content-Length"); 254 // Sometimes the Content-Length is not given and we have to grab it via HEAD method 255 if (size == 0) { 256 size = getSize(); 257 } 258 meter.setTotalWork(size); 259 } 260 261 entity = response.getEntity(); 262 if (entity != null) { 263 in = entity.getContent(); 264 265 // Download the index file 266 out = NetUtil.getOutputStream(dest); 267 268 byte[] buf = new byte[4096]; 269 int count = in.read(buf); 270 while (-1 != count) { 271 if (meter != null) { 272 meter.incrementWorkDone(count); 273 } 274 out.write(buf, 0, count); 275 count = in.read(buf); 276 } 277 } else { 278 String reason = response.getStatusLine().getReasonPhrase(); 279 // TRANSLATOR: Common error condition: {0} is a placeholder for 280 // the URL of what could not be found. 281 Reporter.informUser(this, JSMsg.gettext("Unable to find: {0}", reason + ':' + uri.getPath())); 282 } 283 } catch (IOException e) { 284 // TRANSLATOR: Common error condition: {0} is a placeholder for the 285 // URL of what could not be found. 286 throw new LucidException(JSMsg.gettext("Unable to find: {0}", uri.toString()), e); 287 } finally { 288 // Close the streams 289 IOUtil.close(in); 290 IOUtil.close(out); 291 } 292 } 293 294 /** 295 * Copy this WebResource to the destination. 296 * 297 * @param dest the destination URI 298 * @throws LucidException when an error is encountered 299 */ 300 public void copy(URI dest) throws LucidException { 301 copy(dest, null); 302 } 303 304 /** 305 * Get the field as a long. 306 * 307 * @param response The response from the request 308 * @param field the header field to check 309 * @return the int value for the field 310 */ 311 private int getHeaderAsInt(HttpResponse response, String field) { 312 Header header = response.getFirstHeader(field); 313 // If there is no matching header in the message null is returned. 314 if (header == null) { 315 return 0; 316 } 317 318 String value = header.getValue(); 319 try { 320 return Integer.parseInt(value); 321 } catch (NumberFormatException ex) { 322 return 0; 323 } 324 } 325 326 /** 327 * Get the number of seconds since start of epoch for the field in the response headers as a Date. 328 * 329 * @param response The response from the request 330 * @param field the header field to check 331 * @return number of seconds since start of epoch 332 */ 333 private long getHeaderAsDate(HttpResponse response, String field) { 334 Header header = response.getFirstHeader(field); 335 String value = header.getValue(); 336 // This date cannot be readily parsed with DateFormatter 337 return DateUtils.parseDate(value).getTime(); 338 } 339 /** 340 * Define a 750 ms timeout to get a connection 341 */ 342 private static int timeout = 750; 343 344 private URI uri; 345 private CloseableHttpClient client; 346 } 347