python爬虫11--文件存储之非关系型数据库存储MongoDB

import pymongo
# client = pymongo.MongoClient(host=‘localhost‘,port=27017)    #第一种连接
client = pymongo.MongoClient(‘mongodb://localhost:27017‘)   #第二种连接
print(client)

# db = client.test       #第一种指定
db = client[‘test‘]   #第二种指定

# collection = db.user   #第一种指定
collection = db[‘user‘]    #第二种指定

data1 = {‘id‘:‘001‘, ‘name‘:‘aa‘,‘age‘:15}
data2 = {‘id‘:‘002‘, ‘name‘:‘bb‘,‘age‘:16}
data3 = {‘id‘:‘003‘, ‘name‘:‘cc‘,‘age‘:14}
data4 = {‘id‘:‘004‘, ‘name‘:‘dd‘,‘age‘:14}
collection.insert_one(data1)          #插入一条数据
collection.insert_many([data2,data3,data4])   #插入多条数据

# result = collection.find_one({‘name‘:‘aa‘})   #返回单个结果  {‘_id‘: ObjectId(‘5e1903e232a95ccef7703bda‘), ‘id‘: ‘001‘, ‘name‘: ‘aa‘, ‘age‘: 15}  _id为mongodb插入时自动添加的
results = collection.find({‘age‘:14})       #返回生成器对象
for result in results:
    print(result)

from bson.objectid import ObjectId
result = collection.find_one({‘_id‘:ObjectId(‘5e1903e232a95ccef7703bda‘)})

results = collection.find({‘age‘:{‘$lt‘:15}})   #年龄小于15
results = collection.find({‘age‘:{‘$lte‘:15}})   #年龄小于等于15
results = collection.find({‘age‘:{‘$gt‘:15}})   #年龄大于15
results = collection.find({‘age‘:{‘$gte‘:15}})   #年龄大于等于15
results = collection.find({‘age‘:{‘$ne‘:15}})   #年龄不等于15
results = collection.find({‘age‘:{‘$in‘:[15,16]}})   #年龄在15-16之间，包含
results = collection.find({‘age‘:{‘$nin‘:[15,16]}})   #年龄不在15-16之间
for result in results:
    print(result)

results = collection.find({‘name‘:{‘$regex‘:‘^a.*‘}})   #名字以a开头的正则匹配
for result in results:
    print(result)

result = collection.find().count()
result = collection.find({‘age‘:{‘$gt‘:14}}).count()
print(result)

results = collection.find().sort(‘age‘,pymongo.ASCENDING)   #默认升序
results = collection.find().sort(‘age‘,pymongo.DESCENDING)   #降序

results = collection.find().sort(‘age‘).skip(2)  #从第三个开始
results = collection.find().sort(‘age‘).skip(2).limit(1)  #只要第三个

collection.update_one({‘age‘:15},{‘$set‘:{‘age‘:20}})   #update_one()修改一条，‘$set"直接修改
results = collection.update_many({‘age‘:14},{‘$inc‘:{‘age‘:2}})   #update_many()修改多条，‘$inc‘累计修改
print(results.matched_count,results.modified_count)

collection.remove({‘name‘:‘aa‘})
collection.delete_one({‘name‘:‘bb‘})
results = collection.delete_many({‘age‘:16})
print(results.deleted_count)

# 练习：爬取豆瓣电影TOP250，电影名称，评分，推荐语信息并保存到MongoDB中
from pyquery import PyQuery as pq
import requests
import pymongo

url = ‘https://movie.douban.com/top250‘
headers = {
    ‘User-Agent‘: ‘Mozilla/5.0(Windows NT 6.1;Win64;x64)AppleWebKit/537.36(KHTML,like Gecko)Chrome/79.0.3945.88 Safari/537.36‘
}
res = requests.get(url=url,headers=headers)
doc = pq(res.text)
items = doc(‘.info‘).items()
list_all = []
for item in items:
    movies_dict = {}
    name = pq(item.find(‘.hd‘).html()).find(‘span:first-child‘).text()   #此处的find找到所有符合条件的
    score = pq(item.find(‘.star‘).html()).find(‘span:nth-child(2)‘).text()
    comment = item.find(‘.quote‘).text()
    movies_dict[‘name‘] = name
    movies_dict[‘score‘] = score
    movies_dict[‘comment‘] = comment
    list_all.append(movies_dict)

client = pymongo.MongoClient(host=‘localhost‘,port=27017)
db = client.test
collection = db.movies
for movie in list_all:
    collection.insert_one(movie)

results = collection.find()
for result in results:
    print(result)