《Hands-On Large Language Models》阅读笔记(四)

1from datasets import load_dataset
2dataset = load_dataset("maartengr/arxiv_nlp")

1DatasetDict({
2    train: Dataset({
3        features: ['Titles', 'Abstracts', 'Years', 'Categories'],
4        num_rows: 44949
5    })
6})

 1from datasets import load_dataset
 2from sentence_transformers import SentenceTransformer
 3from umap import UMAP  # 依赖 uv add umap-learn
 4from hdbscan import HDBSCAN # 依赖 uv add hdbscan
 5
 6dataset = load_dataset("maartengr/arxiv_nlp")["train"] # [Titles, Abstracts, Years, Categories]
 7
 8abstracts = dataset["Abstracts"]
 9titles = dataset["Titles"]
10
11# text embeddings
12embedding_model = SentenceTransformer("thenlper/gte-small", device="cuda")
13embeddings = embedding_model.encode(abstracts, batch_size=10, device="cuda", show_progress_bar=True)
14print(embeddings.shape)  # (44949, 384)
15
16# dimensionality reduction, 从 384 降到 5
17umap_model = UMAP(n_components=5, min_dist=0.0, metric="cosine", random_state=42)
18reduced_embeddings = umap_model.fit_transform(embeddings)
19print(reduced_embeddings.shape)  # (44949, 5)
20
21# clustering
22hdbscan_model = HDBSCAN(
23    min_cluster_size=50, metric="euclidean", cluster_selection_method="eom"
24).fit(reduced_embeddings)
25clusters = hdbscan_model.labels_  # clusters 中每条数据对应的聚类标签，[1,3,-1,143]
26print(len(set(clusters)))  # 156 个 cluster, cluster {0,1,2,3,...,153,154,-1}

 1import pandas as pd
 2import matplotlib.pyplot as plt
 3
 4# dimensionality reduction, 从 384 降到 2
 5umap_model = UMAP(n_components=2, min_dist=0.0, metric="cosine", random_state=42)
 6reduced_embeddings = umap_model.fit_transform(embeddings)
 7print(reduced_embeddings.shape)  # (44949, 2)
 8
 9hdbscan_model = HDBSCAN(
10    min_cluster_size=50, metric="euclidean", cluster_selection_method="eom"
11).fit(reduced_embeddings)
12clusters = hdbscan_model.labels_
13
14df = pd.DataFrame(reduced_embeddings, columns=["x", "y"])
15df["title"] = titles
16df['cluster'] = [str(c) for c in clusters]
17
18# 选择离群点和非离群点(聚类)
19clusters_df = df.loc[df.cluster != "-1", :]
20outliers_df = df.loc[df.cluster == "-1", :]
21
22# 生成图片
23plt.scatter(outliers_df.x, outliers_df.y, alpha=0.05, s=2, c="grey")
24plt.scatter(
25    clusters_df.x, clusters_df.y, c=clusters_df.cluster.astype(int),
26    alpha=0.6, s=2, cmap="tab20b"
27)
28plt.axis("off")
29
30plt.savefig("cluster_chart.png", dpi=300, bbox_inches="tight", format="png", transparent=True)

1from bertopic import BERTopic
2
3topic_model = BERTopic(
4    embedding_model=embedding_model,
5    umap_model=umap_model,
6    hdbscan_model=hdbscan_model,
7    verbose=True,
8).fit(abstracts, embeddings)

 1# 平面聚焦分布图
 2reduced_embeddings = umap_model.fit_transform(embeddings)
 3
 4fig = topic_model.visualize_documents(
 5    titles,
 6    reduced_embeddings=reduced_embeddings,
 7    width=1800,
 8    hide_annotations=True
 9)
10
11fig.update_layout(font=dict(size=12))
12
13# 条形图
14topic_model.visualize_barchart(title=titles, n_words=10, autoscale=True, top_n_topics=4)

1from bertopic.representation import KeyBERTInspired
2
3representation_model = KeyBERTInspired()
4topic_model.update_topics(abstracts, representation_model=representation_model)
5
6topic_model.visualize_barchart(title=titles, n_words=10, autoscale=True, top_n_topics=4)

1from bertopic.representation import MaximalMarginalRelevance
2
3representation_model = MaximalMarginalRelevance(diversity=0.2)
4topic_model.update_topics(abstracts, representation_model=representation_model)
5
6topic_model.visualize_barchart(title=titles, n_words=10, autoscale=True, top_n_topics=4)

 1import openai
 2from bertopic.representation import OpenAI
 3
 4prompt = """
 5I have a topic that contains the following documents:
 6[DOCUMENTS]
 7
 8The topic is described by the following keywords: [KEYWORDS]
 9
10Based on the information above, extract a short topic label in the following
11format:
12topic: <short topic label>
13"""
14
15client = openai.OpenAI(base_url="http://localhost:11434/v1", api_key="any")
16representation_model = OpenAI(
17    client, model="gemma4:e4b", exponential_backoff=True, chat=True, prompt=prompt,
18    generator_kwargs={"stop": "xxxxx"}
19)
20topic_model.update_topics(abstracts, representation_model=representation_model)

 1{
 2  "messages": [
 3    {
 4      "role": "system",
 5      "content": "You are an assistant that extracts high-level topics from texts."
 6    },
 7    {
 8      "role": "user",
 9      "content": "<包括几篇相关摘要的和关键字的提示词>"
10    }
11  ],
12  "model": "gemma4:e4b",
13  "stop": "\n"
14}