Loadbank(): Showing progress possible?

BlitzMax Forums/BlitzMax Programming/Loadbank(): Showing progress possible?

Grisu

(Posted 2010) [#1]

Hi guys!

I'm using the following function to download a file from the internet. Is it possible to show the current progress of this process?

Function HTTPGetFile:Int(url$, filePath$)
	Url$ = url$.Replace("http://", "http::")
	Local bank:TBank=LoadBank(url)
	If bank <> Null Then 
	   SaveBank bank,CurrentDir()+"\"+filePath$
         Return 1
      Else 
         Return -1
      EndIf
End Function

Thanks
Grisu

degac

(Posted 2010) [#2]

I dont' know how to 'hack' LoadBank but this is what I use

Rem

		lib remote


		ask to a remote server for

		- size of a file 	(1)		FileSizeRemote(name)
		- copy a file		(2)		CopyFileRemote(source,dest,update_callback,buffersize)

		@ based on the works by
		
		(1) KEV
		(2) LEADWERKS

		


End Rem


Import maxgui.drivers

Global win:TGadget=			CreateWindow("",200,200,400,220,Null,WINDOW_TITLEBAR|WINDOW_CENTER)
Global label_main:tgadget = 		CreateLabel("Downloading..." , 10 , 10 , 390,46,win)
Global progbar:TGadget = 		CreateProgBar(10 , 60 , win.ClientWidth() - 20 , 28 , win,1) 

Global label_size:tgadget = 		CreateLabel("Size: " , 10 , win.ClientHeight()-25,80 , 20 , win)
Global label_position:tgadget = 	CreateLabel("Position" , 211 , win.ClientHeight()-25,80 ,20 , win) 
Global label_pos:tgadget = 		CreateLabel("" , 300,win.ClientHeight()-25, 40 , 16 , win) 
Global file_size:Int


Local file$="http://www.blitzbasic.com/file/get.php?file=/Products/demos/BlitzMaxDemo137.exe"
file_size = FileSizeRemote(file) 
SetUpInfo(file,file_size)

If CopyFileRemote(file$,"",DisplayDownloadProgress)
	Notify "Download successful."
Else
	Notify "Download failed."
EndIf

Function SetUpInfo(_name$ , _size:Int) 
	SetGadgetText label_main , "Downloading: " + _name + "..."
	SetGadgetText label_Size , "Size: " + Int(_size / 1024) + " Kb"
End Function

Function DisplayDownloadProgress(progress:Float)
	UpdateProgBar progbar , progress	
	SetGadgetText label_pos,Int(progress*100)+" %"
	If PeekEvent() WaitEvent() 
PollSystem
EndFunction


Function CheckFileRemote:Int(src$) 
	Local temp_src$
	temp_src$=Replace(src,"\","/")
	temp_src=Replace(temp_src,"://","::")
	Local size:Int = FileSizeRemote(src) 
	
	If size < 0 
	'	DebugLog "---Size <0"
		Return False
	End If
	Local stre:TStream = ReadStream(src)
	Local nogood:Int = 1 , line$
	Local trovato:Int
	If stre
		While Not Eof(stre) 
			line = ReadLine(stre)
				If Instr(Lower(line) , "<html>")>0   
				nogood = - 1
			End If
		Wend
		CloseStream stre
	End If

	Return nogood
	
End Function


Function HTTPFindSource:String(url$)
	Local original$=url
	url=url.Replace("http://","")
	Local slashPos:Int = url$.Find("/"), host$, file$
	Local temp_url$
	
	If slashPos <> -1
		host$ = url$[..slashPos]
		file$ = url$[slashPos..]
	Else
		Return -1
	EndIf
'	DebugLog "HOST: "+host
	Local stream:TStream = OpenStream("tcp::" + host$)
	If Not stream 
	'	DebugLog "TCP - No stream"
		Return -1
	End If
	
	stream.WriteLine "HEAD " + file$ + " HTTP/1.0"
	stream.WriteLine "Host: " + host$
	stream.WriteLine ""
	
	While Not Eof(stream)
		Local in$ = stream.ReadLine()
		If in.find("Location:")<>-1
			temp_url=in[in.find(":")+1..].Trim()
			If FileSizeRemote(temp_url)>0 Return temp_url
			temp_url=HTTPFindSource(temp_url)
			If FileSizeRemote(temp_url)>0 Return temp_url
		End If
			
	Wend
	Return original

End Function


Function CopyFileRemote:Int(src$,dst$="",callback:Byte Ptr=Null,event_so:tgadget=Null,bufsize:Int=128,timeout:Int=5000)
	Local url$=src
	If src = "" Return False
	

		
	If filesizeRemote(src) < 0	Return - 1

	src$=Replace(src,"\","/")
	src=Replace(src,"://","::")
	If dst = "" 
		CreateDir "download"
		dst = AppDir$ + "/download/" + StripDir(src)
	End If
	
	Local size:Int , In:TStream , out:TStream , pos:Int
	Local progress:Float
	Local timer:Int=MilliSecs()
	
	If callback = Null
			Return CopyFile(src , dst) 
	Else
	
		'Set up the callback function
		Local DownloadFileUpdateCallback(p#)
		DownloadFileUpdateCallback=callback
		
	
		'Get the file size
		size=FileSizeRemote(url)
		If size=-1 Return False
	
		'Open streams
		out:TStream=WriteFile(dst)
		If Not out Return False
		in:TStream=ReadStream(src)
		If Not in
			out.close()
		'	DebugLog "No stream..."
			Return False 
		EndIf
		
		'Copy file and display progress
		timer=MilliSecs()
		Local oldpos:Int
		Try
			Local buf:Byte[bufSize]
			While Not in.Eof()
				out.WriteBytes buf,in.Read(buf,bufSize)
				pos=pos+bufsize
				progress#=Float(pos)/Float(size)
				DownloadFileUpdateCallback progress
				
				If oldpos<>pos timer=MilliSecs()
		
				oldpos=pos
				PollSystem
					If EventID()=EVENT_GADGETACTION	
						If EventSource()=event_so
						'	DebugLog "CopyFile - OS event"
							EmitEvent CreateEvent( EVENT_GADGETACTION,event_so)
					
							Return False
						End If
					End If
								
				If MilliSecs()>timer+timeout
				'	DebugLog "CopyFile - timeout!"
					Return False
				End If
			Wend
			Catch ex:TStreamWriteException
		EndTry

		'Close the streams
		in.close()
		out.close()
		
		'Verify file size
		If FileSize(dst)<>size
			DeleteFile dst
			Return False
		EndIf
		
		Return True
		
	EndIf
EndFunction

Function Round:Int(val#)
	Local dec#
	dec#=val-Floor(val)
	If dec<0.5 Return Floor(val) Else Return Ceil(val)
EndFunction


Function FileSizeRemote:Int(url$,timeout:Int=5000) 
	url$ = url$.Replace("http://", "")
	Local slashPos:Int = url$.Find("/"), host$, file$
	
	If slashPos <> -1
		host$ = url$[..slashPos]
		file$ = url$[slashPos..]

	Else

		Return -1
	EndIf
	
	Local stream:TStream = OpenStream("tcp::" + host$)
	If Not stream
		Return -1
	End If
	
	stream.WriteLine "HEAD " + file$ + " HTTP/1.0"
	stream.WriteLine "Host: " + host$
	stream.WriteLine ""
	
	Local timer:Int=MilliSecs()
	
	While Not Eof(stream)
		Local in$ = stream.ReadLine()
		If in$.Find("Content-Length:") <> -1
			stream.Close()
			Return Int(in$[in$.Find(":") + 1..].Trim())
		EndIf	
		PollSystem
		If MilliSecs()>timer+timeout
		'	DebugLog "TIMEOUT !!!!"
			Return -1
		End If
	Wend
	
	stream.Close()

	
	Return -1		
End Function

Type mystream Extends TStream
		
	Method ReadLine$() 
		Local timer:Int=MilliSecs()
		Local str$,buf:Byte[1024],p:Int
		Repeat
			Local n:Byte
			If Read( Varptr n,1 )<>1 Exit
			If n=0 Exit
			If n=10 Exit
			If n=13 Continue
			buf[p]=n ; p:+1
			If p<>buf.length Continue
			str:+String.FromBytes(buf,p)
			p = 0
			If MilliSecs()>timer+10000 Exit
		Forever
		If p str:+String.FromBytes(buf,p)
		Return str
	End Method

End Type

degac

(Posted 2010) [#3]

Ok, I'm hacking something... but I have a problem to retrieve the 'correct' position of the stream, or well, the FULL SIZE of the object to download.

This is your example with a 'real' file.
Below there are some changes, but not fully* working at the moment

Local file$="http://www.graphio.net/download/BlitzMaxCompanion.zip"

HTTPGetFile(file,"bmx.zip")

Function HTTPGetFile:Int(url$, filePath$)
	Url$ = url$.Replace("http://", "http::")
	Local bank:TBank=LoadBank(url,displayprogress)
	If bank <> Null Then 
	   SaveBank bank,CurrentDir()+"\"+filePath$
         Return 1
      Else 
         Return -1
      EndIf
End Function


Function DisplayProgress(progress:Float)
Print progress
End Function

This is in bank.bmx

Function LoadBank:TBank( url:Object ,callback:Byte Ptr=NULL)
	Return TBank.Load( url,callback )
End Function

...


	Function Load:TBank( url:Object,callback:Byte Ptr=Null )
		Local stream:TStream=ReadStream( url )
		If Not stream Return
		Local data:Byte[]=LoadByteArray( stream,callback )
		Local bank:TBank=Create( data.length )
		MemCopy bank.Buf(),data,data.length 
		stream.Close
		Return bank
	End Function

This is in STREAM.bmx

Function LoadByteArray:Byte[]( url:Object,callback:Byte Ptr=Null )
	Local stream:TStream=ReadStream( url )
	If Not stream Throw New TStreamReadException
	Local data:Byte[1024],size
	Global progress:Float
	
	Local DownloadFileUpdateCallback(p#)
	DownloadFileUpdateCallback=callback
	
	While Not stream.Eof()
		If size=data.length data=data[..size*3/2]
		size:+stream.Read( (Byte Ptr data)+size,data.length-size )
		progress#=Float(size)/Float(data.length)
 '<---- I need help here...
		if callback DownloadFileUpdateCallback progress
	Wend
	stream.Close
	Return data[..size]
End Function

Grisu

(Posted 2010) [#4]

Thanks for your help degac.

I don't mess around with bmx core functions.
Have tried to create a single superstrict example of your code. But get a strange error message... :(

[EDIT] deleted source code

degac

(Posted 2010) [#5]

You missed a CreateBank
But still remains the problem about the file size...

SuperStrict 

Framework BRL.HTTPStream 
   Import BRL.System
   Import BRL.Threads
   Import BRL.StandardIO


Local file$="http://www.graphio.net/download/BlitzMaxCompanion.zip"

Print "downloading bmx.zip... please wait..."
HTTPGetFile(file,"bmx222.zip")

Function HTTPGetFile:Int(url$, filePath$)
	Url$ = url$.Replace("http://", "http::")
	Local bank:TBank=LoadBank2(url,displayprogress)
	If bank <> Null Then 
	   SaveBank bank,CurrentDir()+"\"+filePath$
         Return 1
      Else 
         Return -1
      EndIf
End Function


Function DisplayProgress(progress:Float)
Print String(progress:Float) 
End Function

Function LoadBank2:TBank( url:Object ,callback:Byte Ptr=Null)
	Return Load2( url,callback )
End Function


Function Load2:TBank( url:Object,callback:Byte Ptr=Null )
		Local stream:TStream=ReadStream( url )
		If Not stream Return Null 
		Local data:Byte[]=LoadByteArray2( stream,callback )
		Local bank:TBank=CreateBank(data.length ) '<--- here
		MemCopy bank.Buf(),data,data.length 
		stream.Close
		Return bank
End Function
	
Function LoadByteArray2:Byte[]( url:Object,callback:Byte Ptr=Null )
Local stream:TStream=ReadStream( url )
If Not stream Throw New TStreamReadException
Local data:Byte[1024],size:Int
Global progress:Float
	
Local DownloadFileUpdateCallback(p#)
DownloadFileUpdateCallback=callback
	
While Not stream.Eof()
		If size=data.length data=data[..size*3/2]
		size:+stream.Read( (Byte Ptr data)+size,data.length-size )
		progress#=Float(size)/Float(data.length) '<---- I need help here...
		If callback DownloadFileUpdateCallback progress
Wend
stream.Close
Return data[..size]
End Function

Grisu

(Posted 2010) [#6]

This one uses a workaround function, perhaps someone finds a simpler solution?

SuperStrict 

Framework BRL.HTTPStream 
   Import BRL.System
   Import BRL.Threads
   Import BRL.StandardIO


Global file$="http://www.graphio.net/download/BlitzMaxCompanion.zip"
Global file_size:Int = HTTPFileSize:Int(file$)

Print "downloading bmx.zip... "+file_size+"... please wait..."
HTTPGetFile(file,"bmx222.zip")

Function HTTPGetFile:Int(url$, filePath$)
	Url$ = url$.Replace("http://", "http::")
	Local bank:TBank=LoadBank2(url,displayprogress)
	If bank <> Null Then 
	   SaveBank bank,CurrentDir()+"\"+filePath$
         Return 1
      Else 
         Return -1
      EndIf
End Function


Function HTTPFileSize:Int(url$)
	url$ = url$.Replace("http://", "")
	Local slashPos:Int = url$.Find("/"), host$, file$
	
	If slashPos <> -1
		host$ = url$[..slashPos]
		file$ = url$[slashPos..]
	Else
		Return -1
	EndIf
	
	Local stream:TStream=OpenStream("tcp::" + host$, 80)
	If Not stream Then Return -1	
	
	stream.WriteLine "HEAD " + file$ + " HTTP/1.0"
	stream.WriteLine "Host: " + host$
	stream.WriteLine ""
	
	While Not Eof(stream)
		Local in$ = stream.ReadLine()

		If in$.Find("Content-Length:") <> -1
			stream.Close()
			Return Int(in$[in$.Find(":") + 1..].Trim())
		EndIf	
	Wend
	
	stream.Close()
	Return -1		
End Function

Function DisplayProgress(progress:Float)
Print String(progress:Float) 
End Function

Function LoadBank2:TBank( url:Object ,callback:Byte Ptr=Null)
	Return Load2( url,callback )
End Function


Function Load2:TBank( url:Object,callback:Byte Ptr=Null )
		Local stream:TStream=ReadStream( url )
		If Not stream Return Null 
		Local data:Byte[]=LoadByteArray2( stream,callback )
		Local bank:TBank=CreateBank(data.length ) '<--- here
		MemCopy bank.Buf(),data,data.length 
		stream.Close
		Return bank
End Function
	
Function LoadByteArray2:Byte[]( url:Object,callback:Byte Ptr=Null )
Local stream:TStream=ReadStream( url )
If Not stream Throw New TStreamReadException
Local data:Byte[1024],size:Int
Global progress:Float
	
Local DownloadFileUpdateCallback(p#)
DownloadFileUpdateCallback=callback
	
While Not stream.Eof()
		If size=data.length data=data[..size*3/2]
		size:+stream.Read( (Byte Ptr data)+size,data.length-size )
		progress#=Float(size)/Float(file_size) '<---- I need help here...
		If callback DownloadFileUpdateCallback progress
Wend
stream.Close
Return data[..size]
End Function

[EDIT] Streamsize() doesn't apply here.. :(