LaTeX Research Paper Template

\documentclass[12pt]{article}

% Encoding & fonts
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}

% Page layout
\usepackage[margin=2.5cm]{geometry}

% Mathematics
\usepackage{amsmath, amssymb, amsthm}
\usepackage{mathtools}

% Figures & tables
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{array}
\usepackage[labelfont=bf, font=small]{caption}
\usepackage{subcaption}
\usepackage{float}

% References & citations
\usepackage[style=numeric-comp, sorting=none, backend=biber]{biblatex}
\addbibresource{references.bib}

% Hyperlinks
\usepackage[hidelinks, colorlinks=false]{hyperref}
\usepackage{cleveref}

% Algorithms
\usepackage[ruled, vlined]{algorithm2e}

% Utilities
\usepackage{microtype}
\usepackage{xcolor}
\usepackage{lipsum}  % remove in production

% -------------------------------------------------------
% Document metadata
% -------------------------------------------------------
\title{%
  A Sub-Quadratic Attention Mechanism for\\
  Long-Document Summarisation
}
\author{%
  Jane Researcher$^{1}$\thanks{Corresponding author: [email protected]} \and
  John Co-Author$^{2}$
}
\date{%
  $^{1}$Department of Computer Science, University of Example\\
  $^{2}$AI Research Lab, Tech Institute\\[6pt]
  \today
}

\begin{document}

\maketitle

\begin{abstract}
We present an efficient attention mechanism that reduces the computational
complexity of transformer self-attention from $\mathcal{O}(n^2)$ to
$\mathcal{O}(n \log n)$, enabling processing of documents with up to
64{,}000 tokens on a single GPU. Experiments on the CNN/DailyMail and
arXiv summarisation benchmarks show a ROUGE-1 improvement of 2.3 points
over the full-attention baseline with 40\% lower memory consumption.
\end{abstract}

\textbf{Keywords:} natural language processing, attention mechanism,
transformers, document summarisation

\section{Introduction}
Long-document understanding remains a core challenge in NLP.
The quadratic memory complexity of standard self-attention
($\mathcal{O}(n^2 d)$ for sequence length $n$ and dimension $d$)
limits practical sequence lengths to 2{,}048–4{,}096 tokens on
commodity hardware~\cite{vaswani2017}.

\subsection{Contributions}
\begin{enumerate}
  \item A hierarchical attention scheme operating at sentence and paragraph levels.
  \item An open-source implementation evaluated on three public benchmarks.
  \item A theoretical analysis proving the $\mathcal{O}(n \log n)$ bound.
\end{enumerate}

\section{Related Work}
Sparse attention patterns were introduced by~\cite{child2019} and extended
by Longformer~\cite{beltagy2020} with sliding-window plus global tokens.
Linear attention approximations~\cite{katharopoulos2020} achieve $\mathcal{O}(n)$
complexity but sacrifice expressiveness on local patterns.

\section{Methodology}
\subsection{Hierarchical Attention}
Let $\mathbf{X} \in \mathbb{R}^{n \times d}$ be the token embeddings.
We partition $\mathbf{X}$ into $k$ sentence blocks and compute:
\begin{equation}
  \mathbf{H}_i = \text{Attention}\bigl(\mathbf{Q}_i,\, \mathbf{K}_i,\, \mathbf{V}_i\bigr),
  \quad i = 1, \ldots, k
  \label{eq:local-attn}
\end{equation}
where each block attends only within its sentence boundary for local features,
followed by a cross-block pooling step for global context.

\subsection{Complexity Analysis}
For $k$ blocks each of size $m = n/k$:
\begin{equation}
  \mathcal{C} = k \cdot \mathcal{O}(m^2) + \mathcal{O}(k^2)
               = \mathcal{O}\!\left(\frac{n^2}{k}\right) + \mathcal{O}(k^2)
\end{equation}
Setting $k = n^{2/3}$ minimises total complexity to $\mathcal{O}(n^{4/3})$,
and with $k = \sqrt{n}$ we achieve $\mathcal{O}(n \log n)$ in expectation.

\section{Experiments}
\subsection{Datasets and Metrics}
We evaluate on CNN/DailyMail~\cite{see2017} and arXiv~\cite{cohan2018}.
Summaries are scored with ROUGE-1, ROUGE-2, and ROUGE-L.

\begin{table}[H]
  \centering
  \caption{ROUGE scores on CNN/DailyMail test set.}
  \label{tab:results}
  \begin{tabular}{lccc}
    \toprule
    Model              & R-1  & R-2  & R-L  \\
    \midrule
    Lead-3 baseline    & 40.4 & 17.7 & 36.7 \\
    Longformer         & 44.2 & 21.3 & 41.0 \\
    Full Attention     & 45.1 & 21.9 & 41.8 \\
    \textbf{Ours}     & \textbf{47.4} & \textbf{23.1} & \textbf{43.5} \\
    \bottomrule
  \end{tabular}
\end{table}

\section{Conclusion}
We demonstrated that hierarchical attention matches or exceeds full-attention
performance at substantially lower computational cost. Future work will
explore integration with retrieval-augmented generation pipelines.

\printbibliography

\end{document}