|
|
@@ -0,0 +1,206 @@
|
|
|
+{
|
|
|
+ "cells": [
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 1,
|
|
|
+ "id": "initial_id",
|
|
|
+ "metadata": {
|
|
|
+ "collapsed": true,
|
|
|
+ "ExecuteTime": {
|
|
|
+ "end_time": "2024-10-26T12:03:33.521740500Z",
|
|
|
+ "start_time": "2024-10-26T12:03:33.323157Z"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "import requests\n",
|
|
|
+ "from bs4 import BeautifulSoup\n",
|
|
|
+ "import os\n",
|
|
|
+ "from urllib.parse import urljoin\n",
|
|
|
+ "from urllib.parse import urlparse\n",
|
|
|
+ "\n",
|
|
|
+ "# 爬取 https://www.responsibilityreports.com/Companies? 相关公司名称\n",
|
|
|
+ "# url: 网页 url \n",
|
|
|
+ "# headers: 请求头\n",
|
|
|
+ "# limit: 爬取条数 \n",
|
|
|
+ "# TODO\n",
|
|
|
+ "def crawling_company_names(url, headers, limit):\n",
|
|
|
+ " return \n",
|
|
|
+ "# 下载 ESG 报告\n",
|
|
|
+ "def download_pdf(pdf_url, output_folder, headers):\n",
|
|
|
+ " # 获取PDF内容，带上headers和cookie\n",
|
|
|
+ " response = requests.get(pdf_url, headers=headers)\n",
|
|
|
+ "\n",
|
|
|
+ " # 检查请求是否成功\n",
|
|
|
+ " if response.status_code == 200:\n",
|
|
|
+ " # 获取文件名\n",
|
|
|
+ " filename = os.path.join(output_folder, pdf_url.split('/')[-1])\n",
|
|
|
+ "\n",
|
|
|
+ " # 将PDF保存到本地\n",
|
|
|
+ " with open(filename, 'wb') as f:\n",
|
|
|
+ " f.write(response.content)\n",
|
|
|
+ " print(f\"ESG 报告 已下载并保存为 {filename}\")\n",
|
|
|
+ " else:\n",
|
|
|
+ " print(f\"无法下载 ESG 报告，状态码: {response.status_code}\")\n",
|
|
|
+ "\n",
|
|
|
+ "# 获取 ESG 路径\n",
|
|
|
+ "def fetch_esg_pdfs(url, output_folder, headers, cookie):\n",
|
|
|
+ " # 发起请求获取网页内容，带上headers\n",
|
|
|
+ " headers.update({'cookie': cookie})\n",
|
|
|
+ " response = requests.get(url, headers=headers)\n",
|
|
|
+ " parsed_url = urlparse(url)\n",
|
|
|
+ " base_url = parsed_url.scheme + \"://\" + parsed_url.hostname\n",
|
|
|
+ "\n",
|
|
|
+ " if response.status_code == 200:\n",
|
|
|
+ " # 使用BeautifulSoup解析网页内容\n",
|
|
|
+ " soup = BeautifulSoup(response.content, 'html.parser')\n",
|
|
|
+ "\n",
|
|
|
+ " # 找到所有a标签，筛选href属性指向PDF文件的\n",
|
|
|
+ " links = soup.find_all('a', href=True)\n",
|
|
|
+ "\n",
|
|
|
+ " # 创建存储目录（如果不存在）\n",
|
|
|
+ " if not os.path.exists(output_folder):\n",
|
|
|
+ " os.makedirs(output_folder)\n",
|
|
|
+ "\n",
|
|
|
+ " # 遍历所有链接\n",
|
|
|
+ " for link in links:\n",
|
|
|
+ " href = link['href']\n",
|
|
|
+ " if href.endswith('.pdf'): # 检查是否是PDF文件\n",
|
|
|
+ " # 如果链接是相对路径，将其转换为绝对路径\n",
|
|
|
+ " pdf_url = urljoin(base_url, href)\n",
|
|
|
+ " print(f\"发现 ESG 报告: {pdf_url}\")\n",
|
|
|
+ "\n",
|
|
|
+ " # 下载PDF\n",
|
|
|
+ " download_pdf(pdf_url, output_folder, headers)\n",
|
|
|
+ " else:\n",
|
|
|
+ " print(f\"无法访问网页，状态码: {response.status_code}\")\n",
|
|
|
+ "# 定义headers\n",
|
|
|
+ "headers = {\n",
|
|
|
+ " \"accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\",\n",
|
|
|
+ " \"accept-language\": \"zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6\", \"cache-control\": \"no-cache\",\n",
|
|
|
+ " \"priority\": \"u=0, i\",\n",
|
|
|
+ " \"sec-ch-ua\": '\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Microsoft Edge\";v=\"128\"', \"sec-ch-ua-mobile\": \"?0\",\n",
|
|
|
+ " \"sec-ch-ua-platform\": '\"Windows\"', \"sec-fetch-dest\": \"document\", \"sec-fetch-mode\": \"navigate\",\n",
|
|
|
+ " \"sec-fetch-site\": \"none\", \"sec-fetch-user\": \"?1\", \"upgrade-insecure-requests\": \"1\",\n",
|
|
|
+ " \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0\",\n",
|
|
|
+ " 'content-type': 'application/json'}"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "outputs": [
|
|
|
+ {
|
|
|
+ "name": "stdout",
|
|
|
+ "output_type": "stream",
|
|
|
+ "text": [
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2022.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2022.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2022.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2022.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2021.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2021.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2021.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2021.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2020.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2020.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2020.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2020.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2019.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2019.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2019.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2019.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2018.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2018.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2018.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2018.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2017.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2017.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2017.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2017.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2016.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2016.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2016.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2016.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2015.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2015.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2015.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2015.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2014.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2014.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2014.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2014.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2013.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2013.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2013.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2013.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2012.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2012.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2012.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2012.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2011.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2011.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2011.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2011.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2010.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2010.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2010.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2010.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2009.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2009.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2009.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2009.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2008.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2008.pdf\n",
|
|
|
+ "发现 ESG 报告: https://www.responsibilityreports.com/HostedData/ResponsibilityReportArchive/b/TSX_ABX_2008.pdf\n",
|
|
|
+ "ESG 报告 已下载并保存为 ./bgc_esg_reports\\TSX_ABX_2008.pdf\n"
|
|
|
+ ]
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "source": [
|
|
|
+ "url = 'https://www.responsibilityreports.com/Company/barrick-gold-corp' # 目标URL\n",
|
|
|
+ "output_folder = './bgc_esg_reports' # 本地保存路径\n",
|
|
|
+ "cookie = \"_SID_=20241022024928-53ac2a9a2cf61284011d1076f1f5216d; _ga=GA1.1.527769495.1729579762; _ga_FY5TXC48RW=GS1.1.1729579762.1.1.1729579808.0.0.0; _ga_KL4PDK2NVR=GS1.1.1729579764.1.1.1729579808.0.0.0\"\n",
|
|
|
+ "fetch_esg_pdfs(url, output_folder, headers, cookie)"
|
|
|
+ ],
|
|
|
+ "metadata": {
|
|
|
+ "collapsed": false,
|
|
|
+ "ExecuteTime": {
|
|
|
+ "end_time": "2024-10-22T07:07:30.978865200Z",
|
|
|
+ "start_time": "2024-10-22T06:50:41.114191100Z"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "id": "29be7edb2cac126b",
|
|
|
+ "execution_count": 6
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "outputs": [],
|
|
|
+ "source": [],
|
|
|
+ "metadata": {
|
|
|
+ "collapsed": false
|
|
|
+ },
|
|
|
+ "id": "9823135e678caac6"
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "metadata": {
|
|
|
+ "kernelspec": {
|
|
|
+ "display_name": "Python 3",
|
|
|
+ "language": "python",
|
|
|
+ "name": "python3"
|
|
|
+ },
|
|
|
+ "language_info": {
|
|
|
+ "codemirror_mode": {
|
|
|
+ "name": "ipython",
|
|
|
+ "version": 2
|
|
|
+ },
|
|
|
+ "file_extension": ".py",
|
|
|
+ "mimetype": "text/x-python",
|
|
|
+ "name": "python",
|
|
|
+ "nbconvert_exporter": "python",
|
|
|
+ "pygments_lexer": "ipython2",
|
|
|
+ "version": "2.7.6"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "nbformat": 4,
|
|
|
+ "nbformat_minor": 5
|
|
|
+}
|
