import requests
from bs4 import BeautifulSoup
import pymysql
import sqlite3
# 连接MySQL数据库
def connect_mysql():
db = pymysql.connect("localhost","testuser","testpassword","TESTDB")
cursor = db.cursor()
return db, cursor
# 连接SQLite数据库
def connect_sqlite():
db = sqlite3.connect('jobs.sqlite')
cursor = db.cursor()
return db, cursor
# 将数据插入MySQL数据库
def insert_mysql(cursor, job_info):
add_job_sql = """
INSERT INTO jobs_mysql (title, company, location, summary, url)
VALUES (%s, %s, %s, %s, %s)
"""
cursor.execute(add_job_sql, job_info)
# 将数据插入SQLite数据库
def insert_sqlite(cursor, job_info):
add_job_sql = """
INSERT INTO jobs_sqlite (title, company, location, summary, url)
VALUES (?, ?, ?, ?, ?)
"""
cursor.execute(add_job_sql, job_info)
# 爬取招聘信息
def scrape_jobs(url, db_type):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
session = requests.Session()
session.cookies.update({'cookie': 'your_cookie_here'})
response = session.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')
if db_type == 'mysql':
db, cursor = connect_mysql()
elif db_type == 'sqlite':
db, cursor = connect_sqlite()
for job_element in soup.find_all('div', class_='ish-ad-content'):
title = job_element.find('h2', class_='title').text.strip()
company = job_element.find('span', class_='company').text.strip()
location = job_element.find('span', class_='location').text.strip()
summary = job_element.find('div', class_='summary').text.strip()
url = job_element.find('a', class_='ish-ad-link')['href']
job_info = (title, company, location, summary, url)
if db_type == 'mysql':
insert_mysql(cursor, job_info)
db.commit()
elif db_type == 'sqlite':
insert_sqlite(cursor, job_info)
db.commit()
db.close()
# 主函数
def main():
base_url = 'https://www.indeed.com/jobs?q=data+scientist&l=New+York&start='
for i in range(0, 11, 10): # 从第0页爬到第1页,步长为10
url = base_url + str(i)
scrape_jobs(url, 'mysql') # 使用MySQL数据库
scrape_jobs(url, 'sqlite') # 使用SQLite数据库
if __name__ == '__main__':
main()
``