001         package com.croftsoft.core.net;
002    
003         import java.io.*;
004         import java.net.*;
005         import java.util.*;
006    
007         import com.croftsoft.core.lang.StringLib;
008    
009         /*********************************************************************
010         * Downloads the content of a URL to a local File.
011         * 
012         * <p />
013         *
014         * @version
015         *   2003-11-06
016         * @since
017         *   1998-11-30
018         * @author
019         *   <a href="http://www.CroftSoft.com/">David Wallace Croft</a>
020         *********************************************************************/
021    
022         public final class  Downloader
023         //////////////////////////////////////////////////////////////////////
024         //////////////////////////////////////////////////////////////////////
025         {
026    
027         /*********************************************************************
028         * Downloads files.
029         *
030         * <pre>
031         * if ( args.length < 4 )
032         * {
033         *   validate ( args [ 0 ], args [ 1 ], true, true );
034         * }
035         * else
036         * {
037         *   download ( args [ 0 ], args [ 1 ],
038         *     Integer.parseInt ( args [ 2 ] ),
039         *     Integer.parseInt ( args [ 3 ] ) );
040         * }
041         * </pre>
042         *********************************************************************/
043         public static void  main ( String [ ]  args )
044           throws Exception
045         //////////////////////////////////////////////////////////////////////
046         {
047           if ( args.length < 4 )
048           {
049             validate ( args [ 0 ], args [ 1 ], true, true );
050           }
051           else
052           {
053             download ( args [ 0 ], args [ 1 ],
054               Integer.parseInt ( args [ 2 ] ),
055               Integer.parseInt ( args [ 3 ] ) );
056           }
057         }
058    
059         //////////////////////////////////////////////////////////////////////
060         //////////////////////////////////////////////////////////////////////
061    
062         /*********************************************************************
063         * Downloads a sequence of files ending in a number.
064         *********************************************************************/
065         public static void  download (
066           String  urlPrefix,
067           String  filenamePrefix,
068           int     start,
069           int     stop )
070           throws IOException
071         //////////////////////////////////////////////////////////////////////
072         {
073           for ( int  index = start; index <= stop; index++ )
074           {
075             String  url = urlPrefix + index;
076    
077             String  filename = filenamePrefix + index;
078    
079             System.out.println (
080               "Downloading " + url + " to " + filename + "..." );
081    
082             download ( url, filename );
083           }       
084         }
085    
086         public static void  download (
087           String  urlString,
088           String  filename )
089           throws IOException, MalformedURLException
090         //////////////////////////////////////////////////////////////////////
091         {
092           download ( new URL ( urlString ), new File ( filename ) );
093         }
094    
095         /*********************************************************************
096         * Downloads the content at a URL to a local destination File.
097         *********************************************************************/
098         public static void  download ( URL  url, File  dest )
099           throws IOException
100         //////////////////////////////////////////////////////////////////////
101         {
102           download ( url.openStream ( ), dest );
103         }
104    
105         public static void  download (
106           InputStream  unbufferedInputStream,
107           File         destinationFile )
108           throws IOException
109         //////////////////////////////////////////////////////////////////////
110         {
111           BufferedInputStream   in  = null;
112    
113           BufferedOutputStream  out = null;
114    
115           try
116           {
117             in  = new BufferedInputStream ( unbufferedInputStream );
118    
119             out = new BufferedOutputStream (
120               new FileOutputStream ( destinationFile ) );
121    
122             int  i;
123    
124             while ( ( i = in.read ( ) ) > -1 )
125             {
126               out.write ( ( byte ) i );
127             }
128           }
129           finally
130           {
131             if ( out != null )
132             {
133               out.close ( );
134             }
135    
136             if ( in != null )
137             {
138               in.close ( );
139             }
140           }
141         }
142    
143         public static boolean  downloadResourceToDir (
144           URL  codebaseURL, String  name, File  destDir )
145         //////////////////////////////////////////////////////////////////////
146         {
147           if ( ( codebaseURL == null )
148             || ( name        == null )
149             || ( destDir     == null ) )
150    //       || !destDir.exists ( )
151    //       || !destDir.isDirectory ( ) )
152             return false;
153    
154           InputStream           inputStream = null;
155           BufferedInputStream   in          = null;
156           BufferedOutputStream  out         = null;
157    
158           try
159           {
160             // We create the file twice to make sure the path is resolved.
161    //System.out.println ( "destDir:  " + destDir.getCanonicalPath ( ) );
162    //System.out.println ( "name:  " + name );
163             File  cacheFile = new File ( destDir, name );
164    //System.out.println ( "cacheFile1:  " + cacheFile.getCanonicalPath ( ) );
165             cacheFile = new File ( cacheFile.getCanonicalPath ( ) );
166    //System.out.println ( "cacheFile2:  " + cacheFile.getCanonicalPath ( ) );
167    
168             File  parentFile = new File ( cacheFile.getParent ( ) );
169    //System.out.println ( "parentFile:  " + parentFile.getCanonicalPath ( ) );
170    //System.out.println ( "mkdirs:  " + parentFile.mkdirs ( ) );
171             parentFile.mkdirs ( );
172    
173             inputStream = downloadResource ( codebaseURL, name );
174             if ( inputStream == null )
175             {
176               System.out.println ( "  Download of \"" + name + "\" from\n"
177                 + "    \"" + codebaseURL + "\" failed." );
178               return false;
179             }
180    
181             in  = new BufferedInputStream ( inputStream );
182             out = new BufferedOutputStream (
183               new FileOutputStream ( cacheFile ) );
184    
185             int  i;
186             while ( ( i = in.read ( ) ) > -1 ) out.write ( ( byte ) i );
187    
188    System.out.println ( "  Successfully downloaded and saved." );
189    
190             return true;
191           }
192           catch ( Exception  ex )
193           {
194             ex.printStackTrace ( );
195           }
196           finally
197           {
198             try { out.close          ( ); } catch ( Exception  ex1 ) { }
199             try { in.close           ( ); } catch ( Exception  ex1 ) { }
200             try { inputStream.close  ( ); } catch ( Exception  ex1 ) { }
201           }
202    
203           return false;
204         }
205    
206         /*********************************************************************
207         * Returns null if the codebaseURL is null.
208         * Returns null upon failure.
209         *********************************************************************/
210         public static InputStream  downloadResource (
211           URL  codebaseURL, String  name )
212         //////////////////////////////////////////////////////////////////////
213         {
214           if ( codebaseURL == null ) return null;
215    
216           InputStream  inputStream = null;
217           
218           try
219           {
220             String  urlName
221               = replaceSpaces ( replaceSeparators ( name ) );
222    
223             URL  url = new URL ( codebaseURL, urlName );
224    System.out.println ( "Downloading \"" + name + "\" from\n  \"" + url + "\"" );
225    
226             URLConnection  urlConnection = url.openConnection ( );
227    
228             inputStream = urlConnection.getInputStream ( );
229    
230             return inputStream;
231           }
232           catch ( IOException  ex )
233           {
234             try { inputStream.close ( ); } catch ( Exception  ex1 ) { }
235             return null;
236           }
237         }
238    
239         /*********************************************************************
240         * Determines if the local file is valid.
241         *
242         * <p>
243         * If both <i>compareContentLength</i> and <i>compareLastModified</i>
244         * are false, this methods simply checks for the existence of the
245         * local file.
246         * </p>
247         *********************************************************************/
248         public static boolean  isValid (
249           URL      sourceURL,
250           File     localFile,
251           boolean  compareContentLength,
252           boolean  compareLastModified )
253           throws IOException, ProtocolException
254         //////////////////////////////////////////////////////////////////////
255         {
256           if ( !localFile.exists ( ) )
257           {
258             return false;
259           }
260    
261           if ( !compareContentLength && !compareLastModified )
262           {
263             return true;
264           }
265    
266           URLConnection  urlConnection = sourceURL.openConnection ( );
267    
268           if ( compareContentLength )
269           {
270             long  localLength = localFile.length ( );
271    
272    //       System.out.println ( "localLength:  " + localLength );
273    
274    // What if this is greater than Integer.MAX_VALUE?
275    
276             int  sourceLength = urlConnection.getContentLength ( );
277    
278    //       System.out.println ( "sourceLength:  " + sourceLength );
279    
280             if ( localLength != sourceLength )
281             {
282               return false;
283             }
284           }
285    
286           if ( compareLastModified )
287           {
288             long  localLastModified = localFile.lastModified ( );
289    
290             //System.out.println (
291             //  "localLastModified:  " + new Date ( localLastModified ) );
292    
293             long  sourceLastModified = urlConnection.getLastModified ( );
294    
295             //System.out.println (
296             //  "sourceLastModified:  " + new Date ( sourceLastModified ) );
297    
298             if ( localLastModified != sourceLastModified )
299             {
300               return false;
301             }
302           }
303    
304           return true;
305         }
306    
307         /*********************************************************************
308         * Determines if the local file is valid.
309         *
310         * <p>
311         * If both <i>compareContentLength</i> and <i>compareLastModified</i>
312         * are false, this methods simply checks for the existence of the
313         * local file.
314         * </p>
315         *
316         * @param  sourceURLName
317         *
318         *   An HTTP URL.
319         *********************************************************************/
320         public static boolean  isValid (
321           String   sourceURLName,
322           String   localFilename,
323           boolean  compareContentLength,
324           boolean  compareLastModified )
325           throws IOException, ProtocolException
326         //////////////////////////////////////////////////////////////////////
327         {
328           return isValid (
329             new URL  ( sourceURLName ),
330             new File ( localFilename ),
331             compareContentLength,
332             compareLastModified );
333         }
334    
335         /*********************************************************************
336         * Replaces every instance of File.separator with a forward slash
337         * character.  This is used for converting a local path name to
338         * a URL path name.
339         *********************************************************************/
340         public static String  replaceSeparators ( String  localPath )
341         //////////////////////////////////////////////////////////////////////
342         {
343           return StringLib.replace ( localPath, File.separator, "/" );
344         }
345    
346         /*********************************************************************
347         * Replaces every instance of space (" ") with the
348         * x-www-form-urlencoded equivalent ("%20").
349         * This is used for converting a remote filename with spaces so that
350         * it can be downloaded via a HTTP request.
351         *********************************************************************/
352         public static String  replaceSpaces ( String  remoteFilename )
353         //////////////////////////////////////////////////////////////////////
354         {
355           return StringLib.replace ( remoteFilename, " ", "%20" );
356         }
357    
358         /*********************************************************************
359         * Downloads a file if the local copy is missing or outdated.
360         *
361         * @return
362         *
363         *   True if a new copy needed to be downloaded.
364         *********************************************************************/
365         public static boolean  validate (
366           String   sourceURLName,
367           String   localFilename,
368           boolean  checkContentLength,
369           boolean  checkLastModified )
370           throws IOException, MalformedURLException
371         //////////////////////////////////////////////////////////////////////
372         {
373           return validate (
374             new URL  ( sourceURLName ),
375             new File ( localFilename ),
376             checkContentLength,
377             checkLastModified );
378         }
379    
380         /*********************************************************************
381         * Downloads a file if the local copy is missing or outdated.
382         *
383         * @return
384         *
385         *   True if a new copy needed to be downloaded.
386         *********************************************************************/
387         public static boolean  validate (
388           URL      sourceURL,
389           File     localFile,
390           boolean  checkContentLength,
391           boolean  checkLastModified )
392           throws IOException
393         //////////////////////////////////////////////////////////////////////
394         {
395           if ( !localFile.exists ( ) )
396           {
397             Downloader.download ( sourceURL, localFile );
398    
399             return true;
400           }
401           else if ( checkContentLength || checkLastModified )
402           {
403             long  localLength = localFile.length ( );
404    
405             //System.out.println ( "localLength:  " + localLength );
406    
407             long  localLastModified = localFile.lastModified ( );
408    
409             //System.out.println (
410             //  "localLastModified:  " + new Date ( localLastModified ) );
411    
412             URLConnection  urlConnection = sourceURL.openConnection ( );
413    
414             int  sourceLength = urlConnection.getContentLength ( );
415    
416             //System.out.println ( "sourceLength:  " + sourceLength );
417    
418             long  sourceLastModified = urlConnection.getLastModified ( );
419    
420             //System.out.println ( "sourceLastModified:  "
421             //  + new Date ( sourceLastModified ) );
422    
423             InputStream  inputStream = null;
424    
425             try
426             {
427               inputStream = urlConnection.getInputStream ( );
428    
429               if ( checkContentLength
430                      && ( sourceLength != localLength )
431                 || checkLastModified
432                      && ( sourceLastModified > localLastModified ) )
433               {
434                 //System.out.println ( "Downloading \"" + sourceURL
435                 //  + "\" to \"" + localFile + "\"..." );
436    
437                 download ( inputStream, localFile );
438    
439                 return true;
440               }
441             }
442             finally
443             {
444               if ( inputStream != null )
445               {
446                 inputStream.close ( );
447               }
448             }
449           }
450    
451           return false;
452         }
453         
454         //////////////////////////////////////////////////////////////////////
455         //////////////////////////////////////////////////////////////////////
456    
457         private  Downloader ( ) { }
458    
459         //////////////////////////////////////////////////////////////////////
460         //////////////////////////////////////////////////////////////////////
461         }