使用 Python 建立趨勢發現工具的逐步指南：網頁爬蟲、自然語言處理（情感分析與主題建模）及文字雲視覺化

import requests
from bs4 import BeautifulSoup

# 要抓取的網址列表
urls = [“https://en.wikipedia.org/wiki/Natural_language_processing”,
“https://en.wikipedia.org/wiki/Machine_learning”]

collected_texts = [] # 用來存儲每個頁面的文本

for url in urls:
response = requests.get(url, headers={“User-Agent”: “Mozilla/5.0”})
if response.status_code == 200:
soup = BeautifulSoup(response.text, ‘html.parser’)
# 提取所有段落文本
paragraphs = [p.get_text() for p in soup.find_all(‘p’)]
page_text = ” “.join(paragraphs)
collected_texts.append(page_text.strip())
else:
print(f”無法獲取 {url}”)

import re
import nltk
nltk.download(‘stopwords’)
from nltk.corpus import stopwords

stop_words = set(stopwords.words(‘english’))

cleaned_texts = []
for text in collected_texts:
# 移除非字母字符並將文本轉為小寫
text = re.sub(r'[^A-Za-z\s]’, ‘ ‘, text).lower()
# 移除停用詞
words = [w for w in text.split() if w not in stop_words]
cleaned_texts.append(” “.join(words))

from collections import Counter

# 如果分析整體趨勢，將所有文本合併為一個：
all_text = ” “.join(cleaned_texts)
word_counts = Counter(all_text.split())
common_words = word_counts.most_common(10) # 前10個最常見的單詞
print(“前10個關鍵詞:”, common_words)

!pip install textblob
from textblob import TextBlob

for i, text in enumerate(cleaned_texts, 1):
polarity = TextBlob(text).sentiment.polarity
if polarity > 0.1:
sentiment = “正面 😀”
elif polarity < -0.1:
sentiment = “負面 🙁”
else:
sentiment = “中性 😐”
print(f”文件 {i} 情感: {sentiment} (極性={polarity:.2f})”)

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

# 調整這些參數
vectorizer = CountVectorizer(max_df=1.0, min_df=1, stop_words=”english”)
doc_term_matrix = vectorizer.fit_transform(cleaned_texts)

# 擬合 LDA 以找出主題（例如，3個主題）
lda = LatentDirichletAllocation(n_components=3, random_state=42)
lda.fit(doc_term_matrix)

feature_names = vectorizer.get_feature_names_out()

for idx, topic in enumerate(lda.components_):
print(f”主題 {idx + 1}: “, [vectorizer.get_feature_names_out()[i] for i in topic.argsort()[:-11:-1]])

# 假設你有你的文本數據存儲在 combined_text
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import nltk
from nltk.corpus import stopwords
import re

nltk.download(‘stopwords’)
stop_words = set(stopwords.words(‘english’))

# 預處理和清理文本：
cleaned_texts = []
for text in collected_texts:
text = re.sub(r'[^A-Za-z\s]’, ‘ ‘, text).lower()
words = [w for w in text.split() if w not in stop_words]
cleaned_texts.append(” “.join(words))

# 生成合併文本
combined_text = ” “.join(cleaned_texts)

# 生成字雲
wordcloud = WordCloud(width=800, height=400, background_color=”white”, colormap=’viridis’).generate(combined_text)

# 顯示字雲
plt.figure(figsize=(10, 6)) # <– 修正數字尺寸
plt.imshow(wordcloud, interpolation=’bilinear’)
plt.axis(‘off’)
plt.title(“抓取文本的字雲”, fontsize=16)
plt.show()