当前位置: 首页 > news >正文

免费网站正能量app应用大全北京seo推广优化

免费网站正能量app应用大全,北京seo推广优化,温州网页制作设计,商城网站建设哪家好项目一:爬取天气预报数据 【内容】 在中国天气网(http://www.weather.com.cn)中输入城市的名称,例如输入信阳,进入http://www.weather.com.cn/weather1d/101180601.shtml#input 的网页显示信阳的天气预报,其中101180601是信阳的…

项目一:爬取天气预报数据

【内容】
在中国天气网(http://www.weather.com.cn)中输入城市的名称,例如输入信阳,进入http://www.weather.com.cn/weather1d/101180601.shtml#input
的网页显示信阳的天气预报,其中101180601是信阳的代码,每个城市或者地区都有一个代码。如下图所示,请爬取河南所有城市15天的天气预报数据。
1到7天代码
import requests
from bs4 import BeautifulSoup
import csvheaders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36','Accept-Encoding': 'gzip, deflate'
}
city_list = [101180101,101180901,101180801,101180301,101180501,101181101,101180201,101181201,101181501,101180701,101180601,101181401,101181001,101180401,101181701,101181601,101181301]
city_name_dict = {101180101: "郑州市",101180901: "洛阳市",101180801: "开封市",101180301: "新乡市",101180501: "平顶山市",101181101: "焦作市",101180201: "安阳市",101181201: "鹤壁市",101181501: "漯河市",101180701: "南阳市",101180601: "信阳市",101181401: "周口市",101181001: "商丘市",101180401: "许昌市",101181701: "三门峡市",101181601: "驻马店市",101181301: "濮阳"
}# 创建csv文件
with open('河南地级市7天天气情况.csv', 'w', newline='', encoding='utf-8') as csvfile:csv_writer = csv.writer(csvfile)# 写入表头csv_writer.writerow(['City ID', 'City Name', 'Weather Info'])for city in city_list:city_id = citycity_name = city_name_dict.get(city_id, "未知城市")print(f"City ID: {city_id}, City Name: {city_name}")url = f'http://www.weather.com.cn/weather/{city}.shtml'response = requests.get(headers=headers, url=url)soup = BeautifulSoup(response.content.decode('utf-8'), 'html.parser')# 找到v<div id="7d" class="c7d">标签v_div = soup.find('div', {'id': '7d'})# 提取v<div id="7d" class="c7d">下的天气相关的网页信息weather_info = v_div.find('ul', {'class': 't clearfix'})# 提取li标签下的内容,每个标签下的分行打印,移除打印结果之间的空格weather_list = []for li in weather_info.find_all('li'):weather_list.append(li.text.strip().replace('\n', ''))# 将城市ID、城市名称和天气信息写入csv文件csv_writer.writerow([city_id, city_name, ', '.join(weather_list)])
8到15天的代码
import requests
from bs4 import BeautifulSoup
import csvheaders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36','Accept-Encoding': 'gzip, deflate'
}
city_list = [101180101, 101180901, 101180801, 101180301, 101180501, 101181101, 101180201, 101181201, 101181501,101180701, 101180601, 101181401, 101181001, 101180401, 101181701, 101181601, 101181301]
city_name_dict = {101180101: "郑州市",101180901: "洛阳市",101180801: "开封市",101180301: "新乡市",101180501: "平顶山市",101181101: "焦作市",101180201: "安阳市",101181201: "鹤壁市",101181501: "漯河市",101180701: "南阳市",101180601: "信阳市",101181401: "周口市",101181001: "商丘市",101180401: "许昌市",101181701: "三门峡市",101181601: "驻马店市",101181301: "濮阳"
}
# 创建csv文件
with open('河南地级市8-15天天气情况.csv', 'w', newline='', encoding='utf-8') as csvfile:csv_writer = csv.writer(csvfile)# 写入表头csv_writer.writerow(['City ID', 'City Name', 'Weather Info'])for city in city_list:city_id = citycity_name = city_name_dict.get(city_id, "未知城市")print(f"City ID: {city_id}, City Name: {city_name}")url = f'http://www.weather.com.cn/weather15d/{city}.shtml'response = requests.get(headers=headers, url=url)soup = BeautifulSoup(response.content.decode('utf-8'), 'html.parser')# 找到v<div id="15d" class="c15d">标签v_div = soup.find('div', {'id': '15d'})# 提取v<div id="15d" class="c15d">下的天气相关的网页信息weather_info = v_div.find('ul', {'class': 't clearfix'})# 提取li标签下的信息for li in weather_info.find_all('li'):time = li.find('span', {'class': 'time'}).textwea = li.find('span', {'class': 'wea'}).texttem = li.find('span', {'class': 'tem'}).textwind = li.find('span', {'class': 'wind'}).textwind1 = li.find('span', {'class': 'wind1'}).textcsv_writer.writerow([city_id, city_name, f"时间:{time},天气:{wea},温度:{tem},风向:{wind},风力:{wind1}"])
15天代码
import requests
from bs4 import BeautifulSoup
import csvheaders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36','Accept-Encoding': 'gzip, deflate'
}
city_list = [101180101, 101180901, 101180801, 101180301, 101180501, 101181101, 101180201, 101181201, 101181501,101180701, 101180601, 101181401, 101181001, 101180401, 101181701, 101181601, 101181301]
city_name_dict = {101180101: "郑州市",101180901: "洛阳市",101180801: "开封市",101180301: "新乡市",101180501: "平顶山市",101181101: "焦作市",101180201: "安阳市",101181201: "鹤壁市",101181501: "漯河市",101180701: "南阳市",101180601: "信阳市",101181401: "周口市",101181001: "商丘市",101180401: "许昌市",101181701: "三门峡市",101181601: "驻马店市",101181301: "濮阳"
}# 创建csv文件
with open('河南地级市1-15天天气情况.csv', 'w', newline='', encoding='utf-8') as csvfile:csv_writer = csv.writer(csvfile)# 写入表头csv_writer.writerow(['City ID', 'City Name', 'Weather Info'])for city in city_list:city_id = citycity_name = city_name_dict.get(city_id, "未知城市")print(f"City ID: {city_id}, City Name: {city_name}")# 爬取1-7天天气情况url_7d = f'http://www.weather.com.cn/weather/{city}.shtml'response_7d = requests.get(headers=headers, url=url_7d)soup_7d = BeautifulSoup(response_7d.content.decode('utf-8'), 'html.parser')v_div_7d = soup_7d.find('div', {'id': '7d'})weather_info_7d = v_div_7d.find('ul', {'class': 't clearfix'})weather_list_7d = []for li in weather_info_7d.find_all('li'):weather_list_7d.append(li.text.strip().replace('\n', ''))# 爬取8-15天天气情况url_15d = f'http://www.weather.com.cn/weather15d/{city}.shtml'response_15d = requests.get(headers=headers, url=url_15d)soup_15d = BeautifulSoup(response_15d.content.decode('utf-8'), 'html.parser')v_div_15d = soup_15d.find('div', {'id': '15d'})weather_info_15d = v_div_15d.find('ul', {'class': 't clearfix'})weather_list_15d = []for li in weather_info_15d.find_all('li'):time = li.find('span', {'class': 'time'}).textwea = li.find('span', {'class': 'wea'}).texttem = li.find('span', {'class': 'tem'}).textwind = li.find('span', {'class': 'wind'}).textwind1 = li.find('span', {'class': 'wind1'}).textweather_list_15d.append(f"时间:{time},天气:{wea},温度:{tem},风向:{wind},风力:{wind1}")# 将城市ID、城市名称和天气信息写入csv文件csv_writer.writerow([city_id, city_name, ', '.join(weather_list_7d+weather_list_15d)])

项目二:爬取红色旅游数据

【内容】
   信阳是大别山革命根据地,红色旅游资源非常丰富,爬取http://www.bytravel.cn/view/red/index441_list.html 网页的红色旅游景点,并在地图上标注出来。
相关代码
import requests  # 导入requests库,用于发送HTTP请求
import csv  # 导入csv库,用于处理CSV文件
from bs4 import BeautifulSoup  # 导入BeautifulSoup库,用于解析HTML文档headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0 Safari/537.36','Accept-Encoding': 'gzip, deflate'  # 设置请求头,模拟浏览器访问
}# 创建csv文件并写入表头
csv_file = open('信阳红色景点.csv', 'w', newline='', encoding='utf-8')  # 打开csv文件,以写入模式
csv_writer = csv.writer(csv_file)  # 创建csv写入对象
csv_writer.writerow(['景点名称', '景点简介', '星级', '图片链接'])  # 写入表头# 爬取第一页
url = 'http://www.bytravel.cn/view/red/index441_list.html'  # 定义要爬取的网页URL
response = requests.get(headers=headers, url=url)  # 发送GET请求,获取网页内容
soup = BeautifulSoup(response.content.decode('gbk'), 'html.parser')  # 使用BeautifulSoup解析网页内容target_div = soup.find('div', {'style': 'margin:5px 10px 0 10px'})  # 在解析后的HTML中查找目标divfor div in target_div.find_all('div', {'style': 'margin:2px 10px 0 7px;padding:3px 0 0 0'}):  # 在目标div中查找所有符合条件的子divtitle_element = div.find('a', {'class': 'blue14b'})  # 在子div中查找标题元素if title_element:  # 如果找到了标题元素title = title_element.text  # 获取标题文本else:title = "未找到标题"  # 如果没有找到标题元素,设置默认值Introduction_element = div.find('div', id='tctitletop102')  # 在子div中查找简介元素if Introduction_element:  # 如果找到了简介元素intro = Introduction_element.text.strip().replace("[详细]", "")  # 获取简介文本,去除首尾空格和"[详细]"标记else:intro = "无简介"  # 如果没有找到简介元素,设置默认值star_element = div.find('font', {'class': 'f14'})  # 在子div中查找星级元素if star_element:  # 如果找到了星级元素star = star_element.text  # 获取星级文本else:star = "无星级"  # 如果没有找到星级元素,设置默认值img_url_element = div.find('img', {'class': 'hpic'})  # 在子div中查找图片链接元素if img_url_element:  # 如果找到了图片链接元素img_url = img_url_element['src']  # 获取图片链接else:img_url = "无图片链接"  # 如果没有找到图片链接元素,设置默认值print('景点名称:', title)  # 打印景点名称print('景点简介:', intro)  # 打印景点简介print('星级:', star)  # 打印星级print('图片链接:', img_url)  # 打印图片链接# 将数据写入csv文件csv_writer.writerow([title, intro, star, img_url])  # 将景点名称、简介、星级和图片链接写入csv文件# 爬取第二页到第五页
for page in range(1, 5):  # 遍历第二页到第五页url = f'http://www.bytravel.cn/view/red/index441_list{page}.html'  # 构造每一页的URLresponse = requests.get(headers=headers, url=url)  # 发送GET请求,获取网页内容soup = BeautifulSoup(response.content.decode('gbk'), 'html.parser')  # 使用BeautifulSoup解析网页内容target_div = soup.find('div', {'style': 'margin:5px 10px 0 10px'})  # 在解析后的HTML中查找目标divfor div in target_div.find_all('div', {'style': 'margin:2px 10px 0 7px;padding:3px 0 0 0'}):  # 在目标div中查找所有符合条件的子divtitle_element = div.find('a', {'class': 'blue14b'})  # 在子div中查找标题元素if title_element:  # 如果找到了标题元素title = title_element.text  # 获取标题文本else:title = "未找到标题"  # 如果没有找到标题元素,设置默认值Introduction_element = div.find('div', id='tctitletop102')  # 在子div中查找简介元素if Introduction_element:  # 如果找到了简介元素intro = Introduction_element.text.strip().replace("[详细]", "")  # 获取简介文本,去除首尾空格和"[详细]"标记else:intro = "无简介"  # 如果没有找到简介元素,设置默认值star_element = div.find('font', {'class': 'f14'})  # 在子div中查找星级元素if star_element:  # 如果找到了星级元素star = star_element.text  # 获取星级文本else:star = "无星级"  # 如果没有找到星级元素,设置默认值img_url_element = div.find('img', {'class': 'hpic'})  # 在子div中查找图片链接元素if img_url_element:  # 如果找到了图片链接元素img_url = img_url_element['src']  # 获取图片链接else:img_url = "无图片链接"  # 如果没有找到图片链接元素,设置默认值print('景点名称:', title)  # 打印景点名称print('景点简介:', intro)  # 打印景点简介print('星级:', star)  # 打印星级print('图片链接:', img_url)  # 打印图片链接# 将数据写入csv文件csv_writer.writerow([title, intro, star, img_url])  # 将景点名称、简介、星级和图片链接写入csv文件# 关闭csv文件
csv_file.close()

项目三:豆瓣网爬取top250电影数据

【内容】
运用scrapy框架从豆瓣电影top250网站爬取全部上榜的电影信息,并将电影的名称、评分、排名、一句影评、剧情简介分别保存都mysql 和mongodb 库里面。

douban.py
import scrapy  # 导入scrapy库
from scrapy import Selector, Request  # 从scrapy库中导入Selector和Request类
from scrapy.http import HtmlResponse  # 从scrapy库中导入HtmlResponse类
from ..items import DoubanspidersItem  # 从当前目录下的items模块中导入DoubanspidersItem类class DoubanSpider(scrapy.Spider):  # 定义一个名为DoubanSpider的爬虫类,继承自scrapy.Spidername = 'douban'  # 设置爬虫的名称为'douban'allowed_domains = ['movie.douban.com']  # 设置允许爬取的域名为'movie.douban.com'# start_urls = ['http://movie.douban.com/top250']  # 设置起始URL,但注释掉了,所以不会自动开始爬取def start_requests(self):  # 定义start_requests方法,用于生成初始请求for page in range(10):  # 循环10次,每次生成一个请求,爬取豆瓣电影Top250的前10页数据yield Request(url=f'https://movie.douban.com/top250?start={page * 25}&filt=')  # 使用yield关键字返回请求对象,Scrapy会自动处理请求并调用回调函数def parse(self, response: HtmlResponse, **kwargs):  # 定义parse方法,用于解析响应数据sel = Selector(response)  # 使用Selector类解析响应数据list_items = sel.css('#content > div > div.article > ol > li')  # 使用CSS选择器提取电影列表项for list_item in list_items:  # 遍历电影列表项detail_url = list_item.css('div.info > div.hd > a::attr(href)').extract_first()  # 提取电影详情页的URLmovie_item = DoubanspidersItem()  # 创建一个DoubanspidersItem实例movie_item['name'] = list_item.css('span.title::text').extract_first()  # 提取电影名称movie_item['score'] = list_item.css('span.rating_num::text').extract_first()  # 提取电影评分movie_item['top'] = list_item.css('div.pic em ::text').extract_first()  # 提取电影排名yield Request(  # 使用yield关键字返回请求对象,Scrapy会自动处理请求并调用回调函数url=detail_url, callback=self.parse_movie_info, cb_kwargs={'item': movie_item})def parse_movie_info(self, response, **kwargs):  # 定义parse_movie_info方法,用于解析电影详情页数据movie_item = kwargs['item']  # 获取传入的DoubanspidersItem实例sel = Selector(response)  # 使用Selector类解析响应数据movie_item['comment'] = sel.css('div.comment p.comment-content span.short::text').extract_first()  # 提取电影评论movie_item['introduction'] = sel.css('span[property="v:summary"]::text').extract_first().strip() or ''  # 提取电影简介yield movie_item  # 返回处理后的DoubanspidersItem实例,Scrapy会自动处理并保存结果
items.py

import scrapyclass DoubanspidersItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()# passtop = scrapy.Field()name = scrapy.Field()score = scrapy.Field()introduction = scrapy.Field()comment = scrapy.Field()
pipelines.py
from itemadapter import ItemAdapter
import openpyxl
import pymysqlclass DoubanspidersPipeline:def __init__(self):self.conn = pymysql.connect(host='localhost',port=3306,user='root',password='789456MLq',db='sx_douban250',charset='utf8mb4')self.cursor = self.conn.cursor()self.data = []def close_spider(self,spider):if len(self.data) > 0:self._write_to_db()self.conn.close()def process_item(self, item, spider):self.data.append((item['top'],item['name'],item['score'],item['introduction'],item['comment']))if len(self.data) == 100:self._writer_to_db()self.data.clear()return itemdef _writer_to_db(self):self.cursor.executemany('insert into doubantop250 (top,name,score,introduction,comment)''values (%s,%s,%s,%s,%s)',self.data)self.conn.commit()from pymongo import MongoClientclass MyMongoDBPipeline:def __init__(self):self.client = MongoClient('mongodb://localhost:27017/')self.db = self.client['sx_douban250']self.collection = self.db['doubantop250']self.data = []def close_spider(self, spider):if len(self.data) > 0:self._write_to_db()self.client.close()def process_item(self, item, spider):self.data.append({'top': item['top'],'name': item['name'],'score': item['score'],'introduction': item['introduction'],'comment': item['comment']})if len(self.data) == 100:self._write_to_db()self.data.clear()return itemdef _write_to_db(self):self.collection.insert_many(self.data)self.data.clear()class ExcelPipeline:def __init__(self):self.wb = openpyxl.Workbook()self.ws = self.wb.activeself.ws.title = 'Top250'self.ws.append(('排名','评分','主题','简介','评论'))def open_spider(self,spider):passdef close_spider(self,spider):self.wb.save('豆瓣Top250.xlsx')def process_item(self,item,spider):self.ws.append((item['top'], item['name'], item['score'], item['introduction'], item['comment']))return item
settings.py相关内容修改

http://www.ds6.com.cn/news/21850.html

相关文章:

  • 外贸网站建设价格建设网站的十个步骤
  • 商业网站需要多少钱企业所得税优惠政策
  • 网站建设维护费一年多少钱找索引擎seo
  • 手机收藏网站代码品牌网络seo方案外包
  • 企业网站样式谷歌浏览器官网手机版
  • 新型网站设计南京seo
  • 网站开发留言板app推广方案
  • 昆明网页制作青岛关键词优化平台
  • 如何上传到网站根目录最新收录查询
  • 导航网站容易做吗全国防疫大数据平台
  • 做网站必须会编程吗常见的营销型网站
  • 网站备案名称中国开头宣传软文怎么写
  • 成都住建局官网咨询电话seo软件哪个好
  • 石家庄网站制作系统郑州抖音seo
  • 温州手机网站建设千万不要学网络营销
  • 网站 复制 收录被逆冬seo课程欺骗了
  • 大连建设厅网站宁波seo优化公司排名
  • 醴陵网站建设长春网站建设制作
  • 服务类网站开发生成关键词的软件
  • 受欢迎的天津网站建设seo技术306
  • 榆林网站建设vs0912seo免费工具
  • 广州网站建设年底促销杭州seo公司
  • 网站建设设计服务公司批量关键词调排名软件
  • 门户网站设计技巧推广员是干什么的
  • 平度网站建设注册网站需要多少钱?
  • 网站版权符号网站关键词排名优化电话
  • 想在拼购网站做产品代理推广月入5万
  • 蓝色机械营销型网站谷歌seo运营
  • 一起做陶瓷官方网站免费引流推广方法
  • 哪家做网站公司好网推什么意思