Term Frequency vs Page

Term Frequency vs Page#

Navigation: ← Previous: Emotion Lexicon | Next: Word Clouds →

Thematic keywords as time series, then TF–IDF to show which words actually distinguish each novel.

Keyword trajectories#

For each book we track a handful of plot-bearing words (counts per page, then an 8-page rolling mean). This is closer to close reading than a sentiment score: you can see blood arrive in Dracula, morlock in The Time Machine, and christmas take over the Carol.

import sys
from pathlib import Path
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import plotly.graph_objects as go
import plotly.express as px
from IPython.display import HTML, display
import warnings
warnings.filterwarnings('ignore')
plt.style.use('seaborn-v0_8-whitegrid')

PROJ_DIR = Path('.').resolve()
if not (PROJ_DIR / 'gutenberg_utils.py').exists():
    PROJ_DIR = Path('projects/literary-nlp').resolve()
if str(PROJ_DIR) not in sys.path:
    sys.path.insert(0, str(PROJ_DIR))

from gutenberg_utils import (
    load_pages, book_catalog, title_of, BOOK_COLORS, BOOKS,
    THEMATIC_KEYWORDS, all_stopwords, ensure_nltk_data,
    add_vader_sentiment, add_nrc_emotions, NRC_EMOTIONS,
    keyword_counts, character_mentions, third_label,
)

def display_plotly(fig):
    """Embed Plotly with CDN JS — fig.show() is blank in Jupyter Book HTML."""
    display(HTML(fig.to_html(include_plotlyjs='cdn', full_html=False)))

PAGES = load_pages()
CATALOG = book_catalog()
print(f"Loaded {len(PAGES):,} pages across {PAGES['book_id'].nunique()} books")

Loaded 2,653 pages across 8 books

def keyword_figure(book_id):
    df = keyword_counts(PAGES, book_id)
    words = THEMATIC_KEYWORDS[book_id]
    fig = go.Figure()
    for word in words:
        smooth = df[word].rolling(8, min_periods=1, center=True).mean()
        fig.add_trace(go.Scatter(
            x=df['progress'] * 100, y=smooth, mode='lines', name=word,
        ))
    fig.update_layout(
        title=f'{title_of(book_id)} — keyword frequency vs progress',
        xaxis_title='Progress (%)', yaxis_title='Mentions per page (rolling mean)',
        template='plotly_white', height=420, legend=dict(orientation='h', y=-0.2),
    )
    return fig

for book_id in ['dracula', 'pride_and_prejudice', 'time_machine', 'christmas_carol']:
    display_plotly(keyword_figure(book_id))

Distinctive vocabulary (TF–IDF)#

Raw counts favour function words. TF–IDF down-weights terms that appear in every novel and surfaces the words that are characteristic of one title relative to the other seven. Each book is treated as a single document (all pages concatenated).

from sklearn.feature_extraction.text import TfidfVectorizer

docs, labels = [], []
for book_id, grp in PAGES.groupby('book_id', sort=False):
    docs.append(' '.join(grp['text'].astype(str)))
    labels.append(title_of(book_id))

stops = list(all_stopwords())
vec = TfidfVectorizer(stop_words=stops, max_features=4000, min_df=1, max_df=0.9)
X = vec.fit_transform(docs)
terms = np.array(vec.get_feature_names_out())

top_n = 12
chosen = []
for row in X.toarray():
    chosen.extend(terms[row.argsort()[::-1][:top_n]])
chosen = list(dict.fromkeys(chosen))[:40]
idx = [np.where(terms == t)[0][0] for t in chosen]
heat = pd.DataFrame(X.toarray()[:, idx], index=labels, columns=chosen)

fig = px.imshow(
    heat, color_continuous_scale='YlOrRd', aspect='auto',
    labels=dict(color='TF–IDF'),
    title='Distinctive terms (TF–IDF) across the eight novels',
)
fig.update_layout(template='plotly_white', height=560)
fig.update_xaxes(tickangle=45)
display_plotly(fig)

Top five TF–IDF terms per novel:

rows = []
matrix = X.toarray()
for i, label in enumerate(labels):
    order = matrix[i].argsort()[::-1][:5]
    rows.append({'title': label, 'top terms': ', '.join(terms[order])})
pd.DataFrame(rows)
title top terms
0 Dracula van, helsing, lucy, mina, jonathan
1 Wuthering Heights heathcliff, linton, catherine, hareton, earnshaw
2 The Time Machine machine, weena, morlocks, traveller, psychologist
3 Pride and Prejudice elizabeth, darcy, bennet, bingley, jane
4 Frankenstein elizabeth, father, clerval, justine, felix
5 Alice's Adventures in Wonderland alice, turtle, hatter, gryphon, queen
6 A Christmas Carol scrooge, ghost, bob, christmas, spirit
7 The Picture of Dorian Gray dorian, gray, harry, henry, basil

Navigation: ← Previous: Emotion Lexicon | Next: Word Clouds →

Contents