eth/omniexplorer4.py at master · misstop/eth · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import requests
import json
import logging
import pymysql
import os
import yaml
import time
import datetime


'''
表字段顺序
tx_id, type, time, from, to, amount, property_name, property_id, valid
'''
# 日志配置
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s (filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S',
                    filename='omniexplorer4.log',
                    filemode='a')


# 时间戳转换为时间
def cur_time(timestamp):
    now = time.localtime(timestamp)
    cur_time = time.strftime("%Y-%m-%d %H:%M:%S", now)
    return cur_time


# 解析yaml
cur_path = os.path.dirname(os.path.realpath(__file__))
x = yaml.load(open('%s/config.yml' % cur_path, encoding='UTF-8'))
# 数据库
host = x['DATADBO']['MYSQL']['HOST']
username = x['DATADBO']['MYSQL']['UNAME']
pwd = x['DATADBO']['MYSQL']['PWD']
database = x['DATADBO']['MYSQL']['DNAME']


# 数据库连接
def connect_db():
    logging.info('start to connect mysql')
    db = pymysql.connect('{}'.format(host), '{}'.format(username), '{}'.format(pwd), '{}'.format(database))
    logging.info('connect success')
    return db


def insert_db(db, tx_id, type, time, _from, to, amount, property_name, valid, property_id):
    # 使用cursor()方法获取操作游标
    cursor = db.cursor()

    # SQL 插入语句
    sql = "INSERT INTO omni_explorer_3M VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) on duplicate key update tx_id = values(tx_id)"
    par = (tx_id, type, time, _from, to, amount, property_name, valid, property_id)
    try:
        # 执行sql语句
        cursor.execute(sql, par)
        # 提交到数据库执行
        db.commit()
        logging.info("insert success")
    except Exception as e:
        # Rollback in case there is any error
        logging.error(e)
        db.rollback()


# 关闭数据库
def close_db(db):
    db.close()


# 获取页数
def get_pages():
    data = {'addr': '3MbYQMMmSkC3AgWkj9FMo5LsPTW1zBTwXL'}
    try:
        re = requests.post('https://api.omniexplorer.info/v1/transaction/address/0',
                           data=data, timeout=20)
    except Exception as e:
        logging.error(e)
    detail = json.loads(re.text)
    return detail['pages']


# 循环爬取
def crawl_page(db, i):
    data = {'addr': '3MbYQMMmSkC3AgWkj9FMo5LsPTW1zBTwXL'}
    try:
        re = requests.post('https://api.omniexplorer.info/v1/transaction/address/%s' % i,
                           data=data, timeout=20)
    except Exception as e:
        logging.error(e)
    d = json.loads(re.text)['transactions']
    for _ in d:
        d_time = cur_time(_['blocktime'])
        if 'valid' in _.keys():
            valid = 'CONFIRMED'
        else:
            valid = 'UNCONFIRMED'
        if 'amount' in _.keys():
            amount = _['amount']
        else:
            amount = None
        insert_db(db, _['txid'], _['type'], d_time, _['sendingaddress'], _['referenceaddress'], amount,
                  _['propertyname'], valid, _['propertyid'])


# 启动函数
def run():
    i = get_pages()
    logging.info(i)
    db = connect_db()  # 连接MySQL数据库
    for x in range(i):
        try:
            crawl_page(db, x+1)
            time.sleep(0.1)
        except Exception as e:
            logging.error(e)
            continue
    db.close()

if __name__ == '__main__':
    run()