001 package com.croftsoft.core.net;
002
003 import java.io.*;
004 import java.net.*;
005 import java.util.*;
006
007 import com.croftsoft.core.lang.StringLib;
008
009 /*********************************************************************
010 * Downloads the content of a URL to a local File.
011 *
012 * <p />
013 *
014 * @version
015 * 2003-11-06
016 * @since
017 * 1998-11-30
018 * @author
019 * <a href="https://www.croftsoft.com/">David Wallace Croft</a>
020 *********************************************************************/
021
022 public final class Downloader
023 //////////////////////////////////////////////////////////////////////
024 //////////////////////////////////////////////////////////////////////
025 {
026
027 /*********************************************************************
028 * Downloads files.
029 *
030 * <pre>
031 * if ( args.length < 4 )
032 * {
033 * validate ( args [ 0 ], args [ 1 ], true, true );
034 * }
035 * else
036 * {
037 * download ( args [ 0 ], args [ 1 ],
038 * Integer.parseInt ( args [ 2 ] ),
039 * Integer.parseInt ( args [ 3 ] ) );
040 * }
041 * </pre>
042 *********************************************************************/
043 public static void main ( String [ ] args )
044 throws Exception
045 //////////////////////////////////////////////////////////////////////
046 {
047 if ( args.length < 4 )
048 {
049 validate ( args [ 0 ], args [ 1 ], true, true );
050 }
051 else
052 {
053 download ( args [ 0 ], args [ 1 ],
054 Integer.parseInt ( args [ 2 ] ),
055 Integer.parseInt ( args [ 3 ] ) );
056 }
057 }
058
059 //////////////////////////////////////////////////////////////////////
060 //////////////////////////////////////////////////////////////////////
061
062 /*********************************************************************
063 * Downloads a sequence of files ending in a number.
064 *********************************************************************/
065 public static void download (
066 String urlPrefix,
067 String filenamePrefix,
068 int start,
069 int stop )
070 throws IOException
071 //////////////////////////////////////////////////////////////////////
072 {
073 for ( int index = start; index <= stop; index++ )
074 {
075 String url = urlPrefix + index;
076
077 String filename = filenamePrefix + index;
078
079 System.out.println (
080 "Downloading " + url + " to " + filename + "..." );
081
082 download ( url, filename );
083 }
084 }
085
086 public static void download (
087 String urlString,
088 String filename )
089 throws IOException, MalformedURLException
090 //////////////////////////////////////////////////////////////////////
091 {
092 download ( new URL ( urlString ), new File ( filename ) );
093 }
094
095 /*********************************************************************
096 * Downloads the content at a URL to a local destination File.
097 *********************************************************************/
098 public static void download ( URL url, File dest )
099 throws IOException
100 //////////////////////////////////////////////////////////////////////
101 {
102 download ( url.openStream ( ), dest );
103 }
104
105 public static void download (
106 InputStream unbufferedInputStream,
107 File destinationFile )
108 throws IOException
109 //////////////////////////////////////////////////////////////////////
110 {
111 BufferedInputStream in = null;
112
113 BufferedOutputStream out = null;
114
115 try
116 {
117 in = new BufferedInputStream ( unbufferedInputStream );
118
119 out = new BufferedOutputStream (
120 new FileOutputStream ( destinationFile ) );
121
122 int i;
123
124 while ( ( i = in.read ( ) ) > -1 )
125 {
126 out.write ( ( byte ) i );
127 }
128 }
129 finally
130 {
131 if ( out != null )
132 {
133 out.close ( );
134 }
135
136 if ( in != null )
137 {
138 in.close ( );
139 }
140 }
141 }
142
143 public static boolean downloadResourceToDir (
144 URL codebaseURL, String name, File destDir )
145 //////////////////////////////////////////////////////////////////////
146 {
147 if ( ( codebaseURL == null )
148 || ( name == null )
149 || ( destDir == null ) )
150 // || !destDir.exists ( )
151 // || !destDir.isDirectory ( ) )
152 return false;
153
154 InputStream inputStream = null;
155 BufferedInputStream in = null;
156 BufferedOutputStream out = null;
157
158 try
159 {
160 // We create the file twice to make sure the path is resolved.
161 //System.out.println ( "destDir: " + destDir.getCanonicalPath ( ) );
162 //System.out.println ( "name: " + name );
163 File cacheFile = new File ( destDir, name );
164 //System.out.println ( "cacheFile1: " + cacheFile.getCanonicalPath ( ) );
165 cacheFile = new File ( cacheFile.getCanonicalPath ( ) );
166 //System.out.println ( "cacheFile2: " + cacheFile.getCanonicalPath ( ) );
167
168 File parentFile = new File ( cacheFile.getParent ( ) );
169 //System.out.println ( "parentFile: " + parentFile.getCanonicalPath ( ) );
170 //System.out.println ( "mkdirs: " + parentFile.mkdirs ( ) );
171 parentFile.mkdirs ( );
172
173 inputStream = downloadResource ( codebaseURL, name );
174 if ( inputStream == null )
175 {
176 System.out.println ( " Download of \"" + name + "\" from\n"
177 + " \"" + codebaseURL + "\" failed." );
178 return false;
179 }
180
181 in = new BufferedInputStream ( inputStream );
182 out = new BufferedOutputStream (
183 new FileOutputStream ( cacheFile ) );
184
185 int i;
186 while ( ( i = in.read ( ) ) > -1 ) out.write ( ( byte ) i );
187
188 System.out.println ( " Successfully downloaded and saved." );
189
190 return true;
191 }
192 catch ( Exception ex )
193 {
194 ex.printStackTrace ( );
195 }
196 finally
197 {
198 try { out.close ( ); } catch ( Exception ex1 ) { }
199 try { in.close ( ); } catch ( Exception ex1 ) { }
200 try { inputStream.close ( ); } catch ( Exception ex1 ) { }
201 }
202
203 return false;
204 }
205
206 /*********************************************************************
207 * Returns null if the codebaseURL is null.
208 * Returns null upon failure.
209 *********************************************************************/
210 public static InputStream downloadResource (
211 URL codebaseURL, String name )
212 //////////////////////////////////////////////////////////////////////
213 {
214 if ( codebaseURL == null ) return null;
215
216 InputStream inputStream = null;
217
218 try
219 {
220 String urlName
221 = replaceSpaces ( replaceSeparators ( name ) );
222
223 URL url = new URL ( codebaseURL, urlName );
224 System.out.println ( "Downloading \"" + name + "\" from\n \"" + url + "\"" );
225
226 URLConnection urlConnection = url.openConnection ( );
227
228 inputStream = urlConnection.getInputStream ( );
229
230 return inputStream;
231 }
232 catch ( IOException ex )
233 {
234 try { inputStream.close ( ); } catch ( Exception ex1 ) { }
235 return null;
236 }
237 }
238
239 /*********************************************************************
240 * Determines if the local file is valid.
241 *
242 * <p>
243 * If both <i>compareContentLength</i> and <i>compareLastModified</i>
244 * are false, this methods simply checks for the existence of the
245 * local file.
246 * </p>
247 *********************************************************************/
248 public static boolean isValid (
249 URL sourceURL,
250 File localFile,
251 boolean compareContentLength,
252 boolean compareLastModified )
253 throws IOException, ProtocolException
254 //////////////////////////////////////////////////////////////////////
255 {
256 if ( !localFile.exists ( ) )
257 {
258 return false;
259 }
260
261 if ( !compareContentLength && !compareLastModified )
262 {
263 return true;
264 }
265
266 URLConnection urlConnection = sourceURL.openConnection ( );
267
268 if ( compareContentLength )
269 {
270 long localLength = localFile.length ( );
271
272 // System.out.println ( "localLength: " + localLength );
273
274 // What if this is greater than Integer.MAX_VALUE?
275
276 int sourceLength = urlConnection.getContentLength ( );
277
278 // System.out.println ( "sourceLength: " + sourceLength );
279
280 if ( localLength != sourceLength )
281 {
282 return false;
283 }
284 }
285
286 if ( compareLastModified )
287 {
288 long localLastModified = localFile.lastModified ( );
289
290 //System.out.println (
291 // "localLastModified: " + new Date ( localLastModified ) );
292
293 long sourceLastModified = urlConnection.getLastModified ( );
294
295 //System.out.println (
296 // "sourceLastModified: " + new Date ( sourceLastModified ) );
297
298 if ( localLastModified != sourceLastModified )
299 {
300 return false;
301 }
302 }
303
304 return true;
305 }
306
307 /*********************************************************************
308 * Determines if the local file is valid.
309 *
310 * <p>
311 * If both <i>compareContentLength</i> and <i>compareLastModified</i>
312 * are false, this methods simply checks for the existence of the
313 * local file.
314 * </p>
315 *
316 * @param sourceURLName
317 *
318 * An HTTP URL.
319 *********************************************************************/
320 public static boolean isValid (
321 String sourceURLName,
322 String localFilename,
323 boolean compareContentLength,
324 boolean compareLastModified )
325 throws IOException, ProtocolException
326 //////////////////////////////////////////////////////////////////////
327 {
328 return isValid (
329 new URL ( sourceURLName ),
330 new File ( localFilename ),
331 compareContentLength,
332 compareLastModified );
333 }
334
335 /*********************************************************************
336 * Replaces every instance of File.separator with a forward slash
337 * character. This is used for converting a local path name to
338 * a URL path name.
339 *********************************************************************/
340 public static String replaceSeparators ( String localPath )
341 //////////////////////////////////////////////////////////////////////
342 {
343 return StringLib.replace ( localPath, File.separator, "/" );
344 }
345
346 /*********************************************************************
347 * Replaces every instance of space (" ") with the
348 * x-www-form-urlencoded equivalent ("%20").
349 * This is used for converting a remote filename with spaces so that
350 * it can be downloaded via a HTTP request.
351 *********************************************************************/
352 public static String replaceSpaces ( String remoteFilename )
353 //////////////////////////////////////////////////////////////////////
354 {
355 return StringLib.replace ( remoteFilename, " ", "%20" );
356 }
357
358 /*********************************************************************
359 * Downloads a file if the local copy is missing or outdated.
360 *
361 * @return
362 *
363 * True if a new copy needed to be downloaded.
364 *********************************************************************/
365 public static boolean validate (
366 String sourceURLName,
367 String localFilename,
368 boolean checkContentLength,
369 boolean checkLastModified )
370 throws IOException, MalformedURLException
371 //////////////////////////////////////////////////////////////////////
372 {
373 return validate (
374 new URL ( sourceURLName ),
375 new File ( localFilename ),
376 checkContentLength,
377 checkLastModified );
378 }
379
380 /*********************************************************************
381 * Downloads a file if the local copy is missing or outdated.
382 *
383 * @return
384 *
385 * True if a new copy needed to be downloaded.
386 *********************************************************************/
387 public static boolean validate (
388 URL sourceURL,
389 File localFile,
390 boolean checkContentLength,
391 boolean checkLastModified )
392 throws IOException
393 //////////////////////////////////////////////////////////////////////
394 {
395 if ( !localFile.exists ( ) )
396 {
397 Downloader.download ( sourceURL, localFile );
398
399 return true;
400 }
401 else if ( checkContentLength || checkLastModified )
402 {
403 long localLength = localFile.length ( );
404
405 //System.out.println ( "localLength: " + localLength );
406
407 long localLastModified = localFile.lastModified ( );
408
409 //System.out.println (
410 // "localLastModified: " + new Date ( localLastModified ) );
411
412 URLConnection urlConnection = sourceURL.openConnection ( );
413
414 int sourceLength = urlConnection.getContentLength ( );
415
416 //System.out.println ( "sourceLength: " + sourceLength );
417
418 long sourceLastModified = urlConnection.getLastModified ( );
419
420 //System.out.println ( "sourceLastModified: "
421 // + new Date ( sourceLastModified ) );
422
423 InputStream inputStream = null;
424
425 try
426 {
427 inputStream = urlConnection.getInputStream ( );
428
429 if ( checkContentLength
430 && ( sourceLength != localLength )
431 || checkLastModified
432 && ( sourceLastModified > localLastModified ) )
433 {
434 //System.out.println ( "Downloading \"" + sourceURL
435 // + "\" to \"" + localFile + "\"..." );
436
437 download ( inputStream, localFile );
438
439 return true;
440 }
441 }
442 finally
443 {
444 if ( inputStream != null )
445 {
446 inputStream.close ( );
447 }
448 }
449 }
450
451 return false;
452 }
453
454 //////////////////////////////////////////////////////////////////////
455 //////////////////////////////////////////////////////////////////////
456
457 private Downloader ( ) { }
458
459 //////////////////////////////////////////////////////////////////////
460 //////////////////////////////////////////////////////////////////////
461 }