Mysql到Mongodb的数据转换程序_MySQL

# -*- coding: utf-8 -*-

import sys, os
import multiprocessing
import logging
import random
import time, datetime
import MySQLdb
from MySQLdb import cursors
from pymongo import MongoClient

class Config:
    tables = [&＃39;hs_card&＃39;, &＃39;hs_hero&＃39;, &＃39;hs_set&＃39;, &＃39;hs_skill&＃39;, &＃39;hs_level&＃39;, &＃39;hs_pack&＃39;, &＃39;hs_salesevent&＃39;]
    #tables = [&＃39;hs_card&＃39;, &＃39;hs_hero&＃39;, &＃39;hs_set&＃39;, &＃39;hs_skill&＃39;, &＃39;hs_level&＃39;]
    index = {
             &＃39;hs_card&＃39;: [&＃39;name&＃39;],
    }

class Mysql2Mongo(object):
    mysql_host = &＃39;50.23.4.2&＃39;
    mysql_port = 3306
    mysql_user = "root"
    mysql_pass = "stm123"
    mysql_db = "ccg_alpha"

    mongo_host = &＃39;50.23.4.2&＃39;
    mongo_port = 27017

    cOnn= None
    cursor = None
    mOngo= None
    mOngodb= None

    def __init__(self, logger):
		self.logger = logger

		self.cOnn= self.getMysqlConn()
		self.cursor = self.conn.cursor()

		self.mOngo= MongoClient(host=self.mongo_host, port=self.mongo_port)
		self.mOngodb= self.mongo[&＃39;ccg_alpha&＃39;]



    def getMysqlConn(self):
        return MySQLdb.connect(host=self.mysql_host, port=self.mysql_port, user=self.mysql_user, \
                 passwd=self.mysql_pass, db=self.mysql_db, cursorclass=MySQLdb.cursors.SSCursor)


    def setMongoCollectionDocument(self, table, data):
		if(isinstance(data, dict) == False):
			return False
		else:
			self.mongodb[table].insert(data)


    def getMysqlTableDesc(self, table):
		sql = """desc %s""" % (table)
		n = self.cursor.execute(sql)
		data = self.cursor.fetchall()
		keys = []
		types = []
		for row in data:
			key = str(row[0])
			if(row[1].find(&＃39;int&＃39;) >= 0):
				type = 1
	
			elif (row[1].find(&＃39;char&＃39;) >= 0):
				type = 2
			elif (row[1].find(&＃39;text&＃39;) >= 0):
				type = 2
			elif(row[1].find(&＃39;decimal&＃39;) >= 0):
				type = 3
			else:
				type = 2
			keys.append(key)
			types.append(type)
		return keys, types

    def mysql2Mongo(self, table):
        self.mongodb[table].drop()
        keys, types = self.getMysqlTableDesc(table)

        sql = """select * from  %s order by id asc""" % (table)
        n = self.cursor.execute(sql)
        data = self.cursor.fetchall()
        #print table, keys, types
        for row in data:
			ret = {}
			for k, key in enumerate(keys):
				if key == &＃39;id&＃39;:
					key = &＃39;_id&＃39;
					#ret[key] = int(row[k])
				if(types[k] == 1):
					if row[k]==None:
						ret[key]= 0
						continue
					#print k, key, row
					ret[key] = int(row[k])
				elif(types[k] == 2):
					if row[k]==None:
						ret[key]= &＃39;&＃39;
						continue
					ret[key] = str(row[k])
				elif(types[k] == 3):
					if row[k]==None:
						ret[key]= &＃39;&＃39;
						continue
					ret[key] = float(row[k])
				else:
					if row[k]==None:
						ret[key]= &＃39;&＃39;
						continue
					ret[key] = str(row[k])
			#if(table== &＃39;hs_card&＃39;) or (table== &＃39;hs_hero&＃39;):
				#ret[&＃39;rand&＃39;] = random.random()
			print ret
			self.setMongoCollectionDocument(table, ret)



    def __del__(self):
		self.mongo.close()
		self.cursor.close()
		self.conn.close()

if __name__ == &＃39;__main__&＃39;:
    multiprocessing.log_to_stderr()
    logger = multiprocessing.get_logger()
    logger.setLevel(logging.INFO)

    # args = sys.argv
    t1 = time.time()
    cls = Mysql2Mongo(logger)
    for tb in Config.tables:
		cls.mysql2Mongo(tb)
		
    #index    
    for t, f in Config.index.items():
        pass
    
    print time.time() - t1
    logger.info("done")