Project

General

Profile

« Previous | Next » 

Revision 54597

Added by Sandro La Bruzzo almost 6 years ago

added skip store functionality

View differences:

modules/s3-objectStore-migration/trunk/s3_migration.py
52 52 
	client = MongoClient()
53 53 
	db = client['objectStore']
54 54 
	metadataCollection = db[metadataCollection]
55 
	print skip_store
55 56 
	for item in metadataCollection.find(no_cursor_timeout=True):
56 57 
		obsId = item['obsId']
57 58 
		if obsId not in skip_store:
59 
			destination_collection.create_index([('id',pymongo.ASCENDING)])
58 60 
			exportItemForMongoCollection(obsId, db, bucket, log_file)
59 
			destination_collection =db[ 's3_'+obsId[:36]]
60 
			print "creating Index on ID"
61 
			destination_collection.create_index([('id',pymongo.ASCENDING)])
61 
			destination_collection =db[ 's3_'+obsId[:36]]
62 62 
			done_file.write('{}\n'.format(obsId))
63 63 

  		




  64
  64
  
    

  		




  ......




  84
  84
  
    
	conn = boto.connect_s3(aws_access_key_id = props['access_key'], aws_secret_access_key = props['secret_key'],  host = props['host_base'], calling_format = boto.s3.connection.OrdinaryCallingFormat())


  		




  85
  85
  
    
	bucket = conn.get_bucket(bname, validate=True)


  		




  86
  86
  
    
	log_file = open('s3_migration.log', 'w')


  		




  87
  
  
    
	done_file = open('store_done', 'wb')


  		




  
  87
  
    
	done_file = open('store_done_1', 'wb')


  		




  88
  88
  
    
	start_import('metadataObjectStore',bucket, log_file, done_file, skip_store)


  		




  89
  89
  
    
	log_file.close()


  		












Also available in:  Unified diff