BlitzGet Deluxe

This code has been declared by its author to be Public Domain code.

BlitzGet Deluxe by BlitzSupport

2001

Simple function to allow the downloading of a web-based file to disk. Seems to work really well!

Mini History (latest at top)
--
· Fixed stupid case-sensitivity bug!
· Added very rough header parsing using ugly fixed globals and rough text display hackery, lots of re-writing to do now!
· Checks for completed downloads (minor fix still needed -- see 'To do')
· Calls update function every 100 bytes to allow progress reporting
· Checks headers for correct d/l size

; -----------------------------------------------------------------------------
; BlitzGet Deluxe -- based on Mark Sibly's HTTPGet code...
; -----------------------------------------------------------------------------
; WARNING: WILL OVERWRITE EXISTING FILES OF THE SAME NAME!
; -----------------------------------------------------------------------------

; -----------------------------------------------------------------------------
; To do:
; -----------------------------------------------------------------------------
;
; NEED TO CHECK FOR CUTOFFS and... gulp... RESUME? Currently keeps d/l'ing
; nothing if the download is aborted by the server, until the full byte-count
; has been "downloaded"... maybe I'll just say "d/l bombed out" for now...! :)
;
; Parse headers for:
;	404
;
; More to come...
;
; -----------------------------------------------------------------------------

; -----------------------------------------------------------------------------
; Quick demo... downloads to your Windows "Temp" folder
; -----------------------------------------------------------------------------

AppTitle "BlitzGet Deluxe"
Graphics 640, 480, 0, 2
SetBuffer BackBuffer ()

; SMALL TEST (8K):
	download$	 = "http://www.google.com/images/title_homepage4.gif"			; Google homepage logo
;	download$	 = "http://www.blitzbasic.co.nz/zbrowse/blitzfaq/blitzfaq.html" ; Blitz FAQ

; BIG TEST (360K):
;	download$	 = "http://www.hi-toro.com/mp3/diffusion.mp3"					; Pretty music... :P

; SAVE LOCATION: Windows Temp folder:
	downloadDir$ = SystemProperty ("TempDir") ; "G:\My Documents\Downloads\"

If BlitzGet (download$, downloadDir$, "")
	result$ = "Download Complete!" + Chr (10) + "Saved in " + downloadDir$
Else
	result$ = "Download error!"
EndIf

RuntimeError result$

; Temporary, for quick results...

Global header$
Global bytesToRead
Global date$
Global server$
Global contentType$
Global initialReply$

; -----------------------------------------------------------------------------
; File download function
; -----------------------------------------------------------------------------
; webFile$  -- file to download
; saveDir$  -- directory to download into
; saveFile$ -- filename to save as (use "" to use name of downloaded file automatically)
; -----------------------------------------------------------------------------
; Note that if you just provide a web server address, the document downloaded will
; be named "Unknown file.txt"
; -----------------------------------------------------------------------------

Function BlitzGet (webFile$, saveDir$, saveFile$)

	; -------------------------------------------------------------------------
	; Strip "http://" if provided
	; -------------------------------------------------------------------------
	If Left (webFile$, 7) = "http://" Then webFile$ = Right (webFile$, Len (webFile$) - 7)

	; -------------------------------------------------------------------------
	; Split into hostname and path/filename to download
	; -------------------------------------------------------------------------
	slash = Instr (webFile$, "/")
	If slash
		webHost$ = Left (webFile$, slash - 1)
		webFile$ = Right (webFile$, Len (webFile$) - slash + 1)
	Else
		webHost$ = webFile$
		webFile$ = "/"
	EndIf
		
	; -------------------------------------------------------------------------
	; Add trailing slash to download dir if not given
	; -------------------------------------------------------------------------
	If Right (saveDir$, 1) <> "\" Then saveDir$ = saveDir$ + "\"

	; -------------------------------------------------------------------------
	; Save filename -- get from webFile$ if not provided
	; -------------------------------------------------------------------------
	If saveFile$ = ""
		If webFile = "/"
			saveFile$ = "Unknown file.txt"
		Else
			For findSlash = Len (webFile$) To 1 Step - 1
				testForSlash$ = Mid (webFile$, findSlash, 1)
				If testForSlash$ = "/"
					saveFile$ = Right (webFile$, Len (webFile$) - findSlash)
					Exit
				EndIf
			Next
			If saveFile$ = "" Then saveFile$ = "Unknown file.txt"
		EndIf
	EndIf

	; DEBUG
	; RuntimeError "Web host: " + webHost$ + Chr (10) + "Web file: " + webFile$ + Chr (10) + "Save dir: " + saveDir$ + Chr (10) + "Save file: " + saveFile$

	www = OpenTCPStream (webHost$, 80)

	If www
	
		WriteLine www, "GET " + webFile$ + " HTTP/1.1" ; GET / gets default page...
		WriteLine www, "Host: " + webHost$
		WriteLine www, "User-Agent: BlitzGet Deluxe"
		WriteLine www, "Accept: */*"
		WriteLine www, ""
		
		; ---------------------------------------------------------------------
		; Find blank line after header data, where the action begins...
		; ---------------------------------------------------------------------
				
		Repeat

			Cls
			
			header$ = ReadLine (www)

			reply$ = ""
			pos = Instr (header$, ": ")
			If pos
				reply$ = Left (header$, pos + 1)
			EndIf

			Select Lower (reply$)
				Case "content-length: "
					bytesToRead = ReplyContent (header$, reply$)
				Case "date: "
					date$ = ReplyContent (header$, reply$)
				Case "server: "
					server$ = ReplyContent (header$, reply$)
				Case "content-type: "
					contentType$ = ReplyContent (header$, reply$)
				Default
					If gotReply = 0 Then initialReply$ = header$: gotReply = 1
			End Select

			DisplayResponse ()

			Flip
			
		Until header$ = "" Or (Eof (www))
				
		If bytesToRead = 0 Then Goto skipDownLoad
		
		; ---------------------------------------------------------------------
		; Create new file to write downloaded bytes into
		; ---------------------------------------------------------------------
		save = WriteFile (saveDir$ + saveFile$)
		If Not save Then Goto skipDownload

		; ---------------------------------------------------------------------
		; Incredibly complex download-to-file routine...
		; ---------------------------------------------------------------------

		For readWebFile = 1 To bytesToRead
		
			If Not Eof (www) Then WriteByte save, ReadByte (www)
			
			; Call BytesReceived with position and size every 100 bytes (slows down a LOT with smaller updates)
			
			tReadWebFile = readWebFile
			If tReadWebFile Mod 100 = 0 Then BytesReceived (readWebFile, bytesToRead)

		Next

		CloseFile save
		
		; Fully downloaded?
		If (readWebFile - 1) = bytesToRead
			success = 1
		EndIf
		
		; Final update (so it's not rounded to nearest 100 bytes!)
		BytesReceived (bytesToRead, bytesToRead)
		
		.skipDownload
		CloseTCPStream www
		
	Else
	
		RuntimeError "Failed to connect"
		
	EndIf
	
	Return success
	
End Function

; -----------------------------------------------------------------------------
; User-defined update function, called every 100 bytes of download -- alter to suit!
; -----------------------------------------------------------------------------
; TIP: Pass a user-defined type instead, with all data (this stuff plus URL, local filename, etc)
; -----------------------------------------------------------------------------
Function BytesReceived (posByte, totalBytes)
	; Example update code...
	Cls
	Text 0, 10, "Downloading file -- please wait..."
	Text 0, 30, "Received: " + posByte + "/" + totalBytes + " bytes (" + Percent (posByte, totalBytes) + "%)"
	DisplayResponse ()
	Flip
End Function

; -----------------------------------------------------------------------------
; Handy percentage function
; -----------------------------------------------------------------------------
Function Percent (part#, total#)
	Return Int (100 * (part / total))
End Function

Function ReplyContent$ (header$, reply$)
	Return Right (header$, Len (header$) - Len (reply$))
End Function

; Temporary, for quick results...

Function DisplayResponse ()
	Text 0, 80, "Header: " + initialReply$
	Text 0, 100, "Date: " + date$
	Text 0, 120, "Server: " + server$
	Text 0, 140, "Content-Type: " + contentType$
	Text 0, 160, "Content-Length: " + bytesToRead
End Function

Comments

Picklesworth

2005

This code snippet is extremely useful.
I have used it in two seperate projects so far, and I will probably use it in a third as well.

Thanks for writing it!

Imphenzia

2007

Great piece of code - thank you for sharing!

I also suggest the following modification if you want to download any larger files.

Replace

		For readWebFile = 1 To bytesToRead
		
			If Not Eof (www) Then WriteByte save, ReadByte (www)
			
			; Call BytesReceived with position and size every 100 bytes (slows down a LOT with smaller updates)
			
			tReadWebFile = readWebFile
			If tReadWebFile Mod 100 = 0 Then BytesReceived (readWebFile, bytesToRead)

		Next

With this code:

		chunk = 10000: readWebFile = 0
		bnkTest = CreateBank(chunk)	
		Repeat
			If readWebFile + chunk > bytesToRead Then chunk = bytesToRead - readWebFile
			ReadBytes bnkTest, www, 0, chunk
			WriteBytes bnkTest, save, 0, chunk
			readWebFile=readWebFile+chunk
			BytesReceived (readWebFile, bytesToRead)
		 Until readWebFile => bytesToRead Or KeyHit(1)		
		FreeBank (bnkTest)

(note! the function will then return an error code unless you change the line below from:

If (readWebFile -1) = bytesToRead

If readWebFile = bytesToRead

This will work with chunks of data to read and then write to the file rather than single bytes. I just tested it by downloading 6MB in 12 seconds on my connection.

BlitzSupport

2009

In response to RifRaf's query regarding re-direction, here's a little hack (and the original version was really just a quick hack!) to handle, well, re-directed files...

[NB. Uses example from first comment to this post here... which may not be around forever!]

; -----------------------------------------------------------------------------
; BlitzGet Deluxe -- based on Mark Sibly's HTTPGet code...
; -----------------------------------------------------------------------------
; WARNING: WILL OVERWRITE EXISTING FILES OF THE SAME NAME!
; -----------------------------------------------------------------------------

; -----------------------------------------------------------------------------
; To do:
; -----------------------------------------------------------------------------
;
; NEED TO CHECK FOR CUTOFFS and... gulp... RESUME? Currently keeps d/l'ing
; nothing if the download is aborted by the server, until the full byte-count
; has been "downloaded"... maybe I'll just say "d/l bombed out" for now...! :)
;
; Parse headers for:
;	404
;
; More to come...
;
; -----------------------------------------------------------------------------

; -----------------------------------------------------------------------------
; Quick demo... downloads to your Windows "Temp" folder
; -----------------------------------------------------------------------------

AppTitle "BlitzGet Deluxe"
Graphics 640, 480, 0, 2
SetBuffer BackBuffer ()

; SMALL TEST (8K):
	download$	 = "http://www.google.com/images/title_homepage4.gif"			; Google homepage logo
;	download$	 = "http://www.blitzbasic.co.nz/zbrowse/blitzfaq/blitzfaq.html" ; Blitz FAQ

; BIG TEST (360K):
;	download$	 = "http://www.hi-toro.com/mp3/diffusion.mp3"					; Pretty music... :P

download$ = "http://www.rentnet.com/"
;download$ = "http://www.move.com/apartments/main.aspx"

; SAVE LOCATION: Windows Temp folder:
	downloadDir$ = SystemProperty ("TempDir") ; "G:\My Documents\Downloads\"

If BlitzGet (download$, downloadDir$, "")
	result$ = "Download Complete!" + Chr (10) + "Saved in " + downloadDir$
Else
	result$ = "Download error!"
EndIf

Print result$

WaitMouse
End

; Temporary, for quick results...

Global header$
Global bytesToRead
Global date$
Global server$
Global contentType$
Global initialReply$

; -----------------------------------------------------------------------------
; File download function
; -----------------------------------------------------------------------------
; webFile$  -- file to download
; saveDir$  -- directory to download into
; saveFile$ -- filename to save as (use "" to use name of downloaded file automatically)
; -----------------------------------------------------------------------------
; Note that if you just provide a web server address, the document downloaded will
; be named "Unknown file.txt"
; -----------------------------------------------------------------------------

Function BlitzGet (webFile$, saveDir$, saveFile$)

	.requestfile ; NEW/MODIFIED 2009 (re-direct support)
	
	; RE-OPEN STREAM!
	
	DebugLog webFile$
	
	; -------------------------------------------------------------------------
	; Strip "http://" if provided
	; -------------------------------------------------------------------------
	If Left (webFile$, 7) = "http://" Then webFile$ = Right (webFile$, Len (webFile$) - 7)

	; -------------------------------------------------------------------------
	; Split into hostname and path/filename to download
	; -------------------------------------------------------------------------
	slash = Instr (webFile$, "/")
	If slash
		webHost$ = Left (webFile$, slash - 1)
		webFile$ = Right (webFile$, Len (webFile$) - slash + 1)
	Else
		webHost$ = webFile$
		webFile$ = "/"
	EndIf
	
	DebugLog webHost$
	DebugLog webFile$
	
	If Not www ; NEW/MODIFIED 2009 (re-direct support)

		; -------------------------------------------------------------------------
		; Add trailing slash to download dir if not given
		; -------------------------------------------------------------------------
		If Right (saveDir$, 1) <> "\" Then saveDir$ = saveDir$ + "\"
	
		; -------------------------------------------------------------------------
		; Save filename -- get from webFile$ if not provided
		; -------------------------------------------------------------------------
		If saveFile$ = ""
			If webFile = "/"
				saveFile$ = "Unknown file.txt"
			Else
				For findSlash = Len (webFile$) To 1 Step - 1
					testForSlash$ = Mid (webFile$, findSlash, 1)
					If testForSlash$ = "/"
						saveFile$ = Right (webFile$, Len (webFile$) - findSlash)
						Exit
					EndIf
				Next
				If saveFile$ = "" Then saveFile$ = "Unknown file.txt"
			EndIf
		EndIf

	EndIf
	
	; DEBUG
	; RuntimeError "Web host: " + webHost$ + Chr (10) + "Web file: " + webFile$ + Chr (10) + "Save dir: " + saveDir$ + Chr (10) + "Save file: " + saveFile$

	www = OpenTCPStream (webHost$, 80)

	If www
	
		WriteLine www, "GET " + webFile$ + " HTTP/1.1" ; GET / gets default page...
		WriteLine www, "Host: " + webHost$
		WriteLine www, "User-Agent: BlitzGet Deluxe"
		WriteLine www, "Accept: */*"
		WriteLine www, ""
		
		; ---------------------------------------------------------------------
		; NEW/MODIFIED 2009 (re-direct support)
		; ---------------------------------------------------------------------
		; Get HTTP response (200 = OK, 404 = not found, etc)...
		; ---------------------------------------------------------------------

		response$ = ReadLine (www)

		DebugLog "Server response: " + response$
		
		If Left (response$, 5) = "HTTP/"
			pos = Instr (response$, " ")
			replycode$ = Mid (response$, pos + 1, 3)
		EndIf

		; ---------------------------------------------------------------------
		; Find blank line after header data, where the action begins...
		; ---------------------------------------------------------------------
		
		; Store various information along the way...
		
		Repeat
	
			Cls
			
			header$ = ReadLine (www)
	
			reply$ = ""
			pos = Instr (header$, ": ")
			If pos
				reply$ = Left (header$, pos + 1)
			EndIf
	
			Select Lower (reply$)
				Case "content-length: "
					bytesToRead = ReplyContent (header$, reply$)
				Case "date: "
					date$ = ReplyContent (header$, reply$)
				Case "server: "
					server$ = ReplyContent (header$, reply$)
				Case "content-type: "
					contentType$ = ReplyContent (header$, reply$)
				Case "location: " ; NEW/MODIFIED 2009 (re-direct support)
					location$ = ReplyContent (header$, reply$) ; NEW/MODIFIED 2009 (re-direct support)
				Default
					If gotReply = 0 Then initialReply$ = header$: gotReply = 1
			End Select
	
			DisplayResponse ()
	
			Flip
			
		Until header$ = "" Or (Eof (www))

		; ---------------------------------------------------------------------
		; NEW/MODIFIED 2009 (re-direct support)
		; ---------------------------------------------------------------------
		
		downloadfile = 0

		Select replycode$

			Case "200" ; File found...

				downloadfile = True
				
			Case "404" ; File not found...
			
				Goto skipDownLoad
			
			Case "301" ; File permanently moved...
			
				webFile$ = location$
				
				CloseTCPStream www
				www = 0 ; NEW/MODIFIED 2009 (re-direct support)

				Goto requestfile
	
			Case "302" ; File temporarily moved...
	
				webFile$ = location$

				CloseTCPStream www
				www = 0 ; NEW/MODIFIED 2009 (re-direct support)

				Goto requestfile
			
			Case "303" ; File moved...

				webFile$ = location$

				CloseTCPStream www
				www = 0 ; NEW/MODIFIED 2009 (re-direct support)

				Goto requestfile

			Case "307" ; Naughty...

				webFile$ = location$

				CloseTCPStream www
				www = 0 ; NEW/MODIFIED 2009 (re-direct support)

				Goto requestfile
				
		End Select

		; ---------------------------------------------------------------------

		If Not downloadfile Then Goto skipDownLoad ; NEW/MODIFIED 2009 (re-direct support)
		
		If bytesToRead = 0 Then Goto skipDownLoad
		
		; ---------------------------------------------------------------------
		; Create new file to write downloaded bytes into
		; ---------------------------------------------------------------------
		save = WriteFile (saveDir$ + saveFile$)
		If Not save Then Goto skipDownload

		; ---------------------------------------------------------------------
		; Incredibly complex download-to-file routine...
		; ---------------------------------------------------------------------

		For readWebFile = 1 To bytesToRead
		
			If Not Eof (www) Then WriteByte save, ReadByte (www)
			
			; Call BytesReceived with position and size every 100 bytes (slows down a LOT with smaller updates)
			
			tReadWebFile = readWebFile
			If tReadWebFile Mod 100 = 0 Then BytesReceived (readWebFile, bytesToRead)

		Next

		CloseFile save
		
		; Fully downloaded?
		If (readWebFile - 1) = bytesToRead
			success = 1
		EndIf
		
		; Final update (so it's not rounded to nearest 100 bytes!)
		BytesReceived (bytesToRead, bytesToRead)
		
		.skipDownload ; NEW/MODIFIED 2009 (re-direct support)
		CloseTCPStream www
		www = 0 ; NEW/MODIFIED 2009 (re-direct support)
		
	Else
	
		RuntimeError "Failed to connect"
		
	EndIf
	
	Return success
	
End Function

; -----------------------------------------------------------------------------
; User-defined update function, called every 100 bytes of download -- alter to suit!
; -----------------------------------------------------------------------------
; TIP: Pass a user-defined type instead, with all data (this stuff plus URL, local filename, etc)
; -----------------------------------------------------------------------------
Function BytesReceived (posByte, totalBytes)
	; Example update code...
	Cls
	Text 0, 10, "Downloading file -- please wait..."
	Text 0, 30, "Received: " + posByte + "/" + totalBytes + " bytes (" + Percent (posByte, totalBytes) + "%)"
	DisplayResponse ()
	Flip
End Function

; -----------------------------------------------------------------------------
; Handy percentage function
; -----------------------------------------------------------------------------
Function Percent (part#, total#)
	Return Int (100 * (part / total))
End Function

Function ReplyContent$ (header$, reply$)
	Return Right (header$, Len (header$) - Len (reply$))
End Function

; Temporary, for quick results...

Function DisplayResponse ()
	Text 0, 80, "Header: " + initialReply$
	Text 0, 100, "Date: " + date$
	Text 0, 120, "Server: " + server$
	Text 0, 140, "Content-Type: " + contentType$
	Text 0, 160, "Content-Length: " + bytesToRead
End Function

Code Archives Forum