【Python】数据处理（mongodb、布隆过滤器、索引）

from pymongo import MongoClient
from bson.objectid import ObjectId
from bitarray import bitarray

# 连接MongoDB
client = MongoClient('mongodb://localhost:27017/')
db = client['mydatabase']
collection = db['mycollection']

# 创建布隆过滤器
filter_bits = 100000
hash_functions = 10

def create_bloom_filter():
    return bitarray(filter_bits)

# 初始化布隆过滤器
bloom_filter = create_bloom_filter()
bloom_filter.setall(0)

# 向布隆过滤器添加元素
def add_to_bloom_filter(bloom_filter, value):
    for i in range(hash_functions):
        index = hash(str(value) + str(i)) % filter_bits
        bloom_filter[index] = 1

# 检查元素是否可能在集合中
def might_exist_in_set(bloom_filter, value):
    exists = True
    for i in range(hash_functions):
        index = hash(str(value) + str(i)) % filter_bits
        if bloom_filter[index] == 0:
            exists = False
            break
    return exists

# 向MongoDB插入数据前，先检查数据是否已存在于布隆过滤器
def insert_if_not_exists(bloom_filter, document):
    if might_exist_in_set(bloom_filter, document['_id']):
        print(f"Document with ID {document['_id']} already exists.")
    else:
        collection.insert_one(document)
        add_to_bloom_filter(bloom_filter, document['_id'])
        print(f"Document with ID {document['_id']} inserted.")

# 示例数据
document = {'_id': ObjectId(), 'name': 'Alice', 'age': 30}

# 插入数据前，先创建布隆过滤器索引
insert_if_not_exists(bloom_filter, document)