Example of python multithreaded download based on queue and threading

This article example describes the python based queue and threading to achieve multi-threaded download method, shared for your reference. Specific methods are as follows:

The main code is as follows:

  #download worker 
  queue_download = (0) 
  DOWNLOAD_WORKERS = 20 
  for i in range(DOWNLOAD_WORKERS): 
    DownloadWorker(queue_download).start() #start a download worker 
     
  for md5 in MD5S: 
    queue_download.put(md5) 
  for i in range(DOWNLOAD_WORKERS): 
    queue_download.put(None)

included among these
Class inheritance, overloading the run method... Called in __init__. __init__(self), __init__(self), __init__(self), __init__(self)
Implementing time-consuming operations in the run method

import threading 
import Queue 
import md5query 
import DOM 
import os,sys 

class DownloadWorker(): 
  """""" 
 

  def __init__(self, queue): 
    """Constructor""" 
    self.__queue = queue 
    .__init__(self) 
 
 
  def run(self): 
    while 1: 
      md5 = self.__queue.get() 
      if md5 is None: 
        break #reached end of queue 
      #this is a time-cost produce 
      self._down(md5) 
 
      print "task:", md5, "finished" 
 
  def _down(self, md5): 
    config = { 
      'input':,  
      'output':'./samples',  
      'location':'xxx',  
      'has-fn':False,  
      'options':{'':60, 'timeout':3600},  
      'log':file('', 'w'),  
    } 
    print 'download %s...' % (md5) 
    try: 
      data = downloadproc(config['location'], config['options'])# My download process
      if data: 
        dom, fileData = (data) 
        filename = md5 
        if config['has-fn']: 
          filename = '%s_%s' % (md5, dom.nodeValue2('xxxxxxx', '').encode('utf-8'))# This is how I downloaded it
        f = file((config['output'], filename), 'w') 
        (fileData) 
        () 
 
        print '%s\tok' % (md5) 
      else: 
        print>>config['log'], '%s\t%s' % (md5, 'failed') 
    except Exception, e: 
      print>>config['log'], '%s\t%s' % (md5, str(e))

I hope that what I have described in this article will help you in your Python programming.