visualtoks/visualtoks-doc.tex at main · TeXackers/visualtoks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
\documentclass[11pt]{article}

\usepackage{geometry}

\usepackage{mlmodern}

\usepackage{enumitem}
\setlist[itemize]{align=parleft,left=0pt..30pt}

\parindent=0pt
\parskip=\medskipamount

\usepackage[hidelinks]{hyperref}

\usepackage{visualtoks}

\protected\def\link{\futurelet\tempa\linkA}
\def\linkA{\ifx[\tempa \expandafter\linkB \else \expandafter\linkC \fi}
\long\def\linkB[#1]#2{\href{#2}{\ttul{#1}}}
\long\def\linkC#1{\href{#1}{\ttul{#1}}}
\protected\long\def\email#1{\href{mailto:#1}{\ttul{#1}}}

\protected\long\def\ttul#1{%
	\leavevmode
	\setbox0=\hbox{\ttfamily #1}%
	\lower2.25pt\rlap{\leaders\hrule height.625px\hskip\wd0}%
	\pdfliteral{q 1 G 2 Tr 1.7 w 1 j 1 J}%
	\rlap{\copy0}%
	\pdfliteral{Q}%
	\box0
}

\title{The \texttt{visualtoks} Package, version 1.1b}
\author{plante}

\begin{document}

\maketitle

\noindent
In The \TeX book, Knuth demonstrates the concept of tokens with the following example:
\begin{quote}
	\medskip
	For example, if the normal conventions of plain \TeX\ are in force, the text
	`\verb|{\hskip 36 pt}|' is converted into a list of eight tokens:
	\[
		\hbox{\visualtokssep=1.25em \visualtoks{{\hskip 36 pt}}}
	\]
	The subscripts here are the category codes, as listed earlier: 1 for ``beginning of group,''
	12 for ``other character,'' and so on. The {\ttfamily \visualtoksB{hskip}} doesn't
	get a subscript, because it represents a control sequence token instead of a character token.
	Notice that the space after \verb|\hskip| does not get into the token list, because it
	follows a control word. \hfill (p.\ 38){\parfillskip=0pt \par}
	\medskip
\end{quote}
The same style of token display is used several times in the \TeX book.
It would be useful to be able to generate the display automatically
for an arbitrary list of tokens, for pedagogical or debugging purposes.
This package provides the \verb|\visualtoks| command which does exactly that.

\subsection*{Usage}

Usage: \verb|\visualtoks{|$\,\langle\textit{token list}\rangle\,$\verb|}|.

This package may be loaded by \verb|\input{visualtoks}| (plain \TeX\ and other formats) or \verb|\usepackage{visualtoks}| (\LaTeX). % The $\varepsilon$-\TeX\ extensions are required for the \verb|\detokenize| primitive.
It depends on the \texttt{tokmap} package.

The horizontal separation between displayed tokens may be configured by the dimen register \verb|\visualtokssep|. The default value is 1em.

$\langle\textit{token list}\rangle$ must be balanced with respect to explicit braces, and must not contain the token \verb|\tokmap@nil|.
It is assumed that \verb|{| and \verb|}| are the only characters with category codes 1 (beginning of group) and 2 (end of group) respectively.

An \emph{anomalous} control sequence is one that differs in shape from the control sequence with the same name constructed by \verb|\csname|. Anomalous control sequences are marked with a star next to their box.

\subsection*{Samples}

\begin{itemize}

\item \verb|\visualtoks{\def \macro#1{abc #1\egroup}}| gives
\[ \hbox{\visualtoks{\def \macro #1{abc #1\egroup}}}. \]

\item \verb|\visualtoks{$$\halign{&##\hfil\crcr}$$\par}| gives
\[ \hbox{\visualtoks{$$\halign{&##\hfil\crcr}$$\par}}. \]

\item Unbalanced \verb|\if...| tokens: \\
	\verb|\visualtoks{\ifnum\iffalse{\fi`} = 0\else}| gives
\[ \hbox{\visualtoks{\ifnum\iffalse{\fi`} = 0\else}}. \]

\makeatletter
\item To demonstrate how \TeX\ tokenizes consecutive spaces: \\
	\verb|\makeatletter |\verb*|\edef\temp{{    }{\@spaces}}| \\
	\verb|\expandafter\visualtoks\expandafter{\temp}| gives
\[ \hbox{\edef\temp{{    }{\@spaces}} \expandafter\visualtoks\expandafter{\temp}}. \]

\item To demonstrate the \verb|\lowercase| technique: \\
	\verb|\begingroup| \\
	\verb|    \lccode`&=`$ \lccode`#=`$ \lccode`^=`$ \lccode`_=`$| \\
	\verb|    \lccode`|\verb*| |\verb|=`$ \lccode`A=`$ \lccode`?=`$ \lccode`~=`$| \\
	\verb|\lowercase{\endgroup\def\temp{$&##^_|\verb*| |\verb|A?~}}| \\
	\verb|\expandafter\visualtoks\expandafter{\temp}| gives
\begingroup
	\lccode`&=`$ \lccode`#=`$ \lccode`^=`$ \lccode`_=`$
	\lccode` =`$ \lccode`A=`$ \lccode`?=`$ \lccode`~=`$
\lowercase{\endgroup\def\temp{$&##^_ A?~}}
\[ \hbox{\expandafter\visualtoks\expandafter{\temp}}. \]

\item To show anomalous tokens: \\
\verb|\font\tenrm=cmr10 \tenrm| \\
\verb|\expandafter\visualtoks\expandafter{\the\font \tenrm}| gives

\[ \hbox{\font\tenrm=cmr10 \tenrm
	\expandafter\visualtoks\expandafter{\the\font \tenrm}}. \]

% \setbox0=\vbox{\halign{#\crcr \global\futurelet\temp\relax \crcr}}
% \expandafter\visualtoks\expandafter{\temp \endtemplate}

% {\obeyspaces\let \fi\visualtoks{   }}

\end{itemize}

\subsection*{License}

This package is copyright \copyright\ 2025 plante, and
released under the \LaTeX\ Project Public License (\textsc{lppl}) 1.3c.

\subsection*{Repository}

The upstream repository of this package may be found at
	\[ \link{https://github.com/texackers/visualtoks/tree/main}. \]

\end{document}