% File: prettytok.sty
% Copyright 2022-2023 user202729
%
% This work  may be  distributed and/or  modified under  the conditions  of the
% LaTeX Project Public License (LPPL),  either version 1.3c  of this license or
% (at your option) any later version.  The latest version of this license is in
% the file:
%
%   http://www.latex-project.org/lppl.txt
%
% This work has the LPPL maintenance status `maintained'.
% 
% The Current Maintainer of this work is user202729.

\ProvidesExplPackage{prettytok}{2023-04-18}{0.2.0}{Pretty-print token list}
\RequirePackage{precattl}
\RequirePackage{l3keys2e}


\keys_define:nn{prettytok}{
	mode.choices:nn={ term-8bit, term-shell, html }{
		\str_set:Nx \_prettytok_mode {\l_keys_choice_tl}
	},
	mode.initial:n = { term-8bit },

	ignore-tlanalysis-bug.bool_set:N=\_prettytok_ignore_tlanalysis_bug,

	term-shell-decode-cmd.tl_set:N=\_prettytok_term_shell_decode_cmd,
	term-shell-decode-cmd.initial:n={},
	term-shell-decode-cmd-print.bool_set:N=\_prettytok_term_shell_decode_cmd_print,

	html-file-name.tl_set:N=\_prettytok_html_file_name,
	html-file-name.initial:x={pretty-\jobname.html},
	html-refresh-strategy.tl_set:N=\_prettytok_html_refresh_strategy,
	html-refresh-strategy.initial:n=4,
	html-refresh-duration.tl_set:N=\_prettytok_html_refresh_duration,
	html-refresh-duration.initial:n=1000,

	term-prefix.tl_set:N=\prettytermprefix,
	term-prefix.initial:n={>~},
	term-prefix-more.tl_set:N=\prettytermprefixmore,
	term-prefix-more.initial:n={>~ ..~},
	term-wrap-limit.tl_set:N=\prettytermwraplimit,
	term-wrap-limit.initial:n=70,

}
\ProcessKeysOptions{prettytok}


% ======== check tl_analysis bug.
\bool_if:NF \_prettytok_ignore_tlanalysis_bug {
	\tl_set:Nn \_prettytok_test {}
	\tl_analysis_map_inline:nn {ab} {
		\tl_analysis_map_inline:nn {cd} {
			\tl_put_right:Nx \_prettytok_test {#1 ##1 .}
		}
	}
	\tl_if_eq:NnF \_prettytok_test {ac.ad.bc.bd.} {
		\msg_new:nnn {prettytok} {tl-analysis-bug}
		{
			Because~of~an~tl-analysis~bug~(see~https://github.com/latex3/latex3/issues/1073),~
			functions~will~not~be~usable~inside~tl-analysis~functions!~
			Upgrade~your~LaTeX~version,~or~
			pass-option~"ignore-tlanalysis-bug"~to~ignore~this~error.
		}
		\msg_error:nn {prettytok} {tl-analysis-bug}
	}
}

% ======== load Lua module.
\sys_if_engine_luatex:T{
	\directlua{require("prettytok")}
	\precattl_exec:n {
		\directlua{prettyprint_frozenrelaxtok=token.get_next().tok}\cFrozenRelax
	}
}

% ======== main code start.
\cs_generate_variant:Nn \exp_args:NNn {NNx}
\cs_generate_variant:Nn \iow_open:Nn {NV}
\cs_generate_variant:Nn \exp_args:NNn {NNV}
\cs_generate_variant:Nn \iow_now:Nn {Nx}
\cs_generate_variant:Nn \tl_if_eq:nnF {xoF}
\cs_generate_variant:Nn \tl_if_eq:nnTF {o}
\cs_generate_variant:Nn \use:N {c}
\cs_generate_variant:Nn \str_map_inline:nn {xn}
\cs_generate_variant:Nn \tl_build_put_right:Nn {Nx}
\cs_generate_variant:Nn \tl_build_put_right:Nn {NV}
\cs_generate_variant:Nn \iow_now:Nn {NV}


\str_if_eq:VnTF \_prettytok_mode {html} {
	\iow_new:N \_prettytok_file
	\ior_new:N \_prettytok_input_template_file

	\iow_open:NV \_prettytok_file \_prettytok_html_file_name

	\precattl_exec:n{
	% write the template
	\ior_open:Nn \_prettytok_input_template_file {prettytok_template.html}
	\ior_str_map_variable:NNn \_prettytok_input_template_file \_prettytok_line {
		\str_replace_all:Nnn \_prettytok_line {\cO\^^I} {~}  % workaround: XeTeX prints literal tab character as ^^I which obviously breaks JavaScript
		\iow_now:NV \_prettytok_file \_prettytok_line
	}
	\ior_close:N \_prettytok_input_template_file

	% refresh strategy
	\iow_now:Nx \_prettytok_file {set_refresh_strategy(\_prettytok_html_refresh_strategy,\_prettytok_html_refresh_duration)}

	\cs_new_protected:Npn \pretty:n #1 {

		% ======== the actual content. Collect to one \write to put them on the same line (also maybe for efficiency???)
		\tl_build_begin:N \_prettytok_content
		\tl_build_put_right:Nn \_prettytok_content {print_tl(}
		\tl_analysis_map_inline:nn {#1} {
			% by documentation of \tl_analysis_map_inline:nn: #1=token, #2=char code, #3=catcode
			\int_compare:nNnTF {##2} = {-1} {
				% token is control sequence
				\tl_if_eq:onTF {##1} {\cFrozenRelax} {
					\tl_build_put_right:Nn \_prettytok_content {csfrozenrelax(),}
				} {
					\tl_build_put_right:Nn \_prettytok_content {cs(}
					\tl_if_eq:xoF {\use:c {}} {##1} {
						\str_map_inline:xn {\exp_after:wN \cs_to_str:N ##1} {  % side note, must use str here to preserve spaces
							\tl_build_put_right:Nx \_prettytok_content {\int_eval:n {`####1},}
						}
					}
					\tl_build_put_right:Nn \_prettytok_content {),}
				}
			}
			{
				% token is not control sequence
				\tl_build_put_right:Nn \_prettytok_content {token(##2, "##3"),}
			}
		}
		\tl_build_put_right:Nn \_prettytok_content {)//</script><script>}
		\tl_build_end:N \_prettytok_content
		\iow_now:NV \_prettytok_file \_prettytok_content
	}
	}

} {
	\str_if_eq:VnTF \_prettytok_mode {term-shell} {
		\newwrite \_prettytok_typeout_stream
		\tl_if_empty:NTF \_prettytok_term_shell_decode_cmd {
			\sys_get_shell:nnN {kpsewhich~ prettytok-decode-8bit.py} {\cctab_select:N \c_other_cctab} \_prettytok_term_shell_decode_cmd
		} {
			\begingroup
				\escapechar=-1~
				\str_gset:NV \_prettytok_term_shell_decode_cmd \_prettytok_term_shell_decode_cmd
			\endgroup
		}
		\bool_if:NT \_prettytok_term_shell_decode_cmd_print {
			\typeout{^^JThe~ value~ of~ term-shell-decode-cmd~ is:~ [[[\_prettytok_term_shell_decode_cmd]]]^^J}
		}
		\immediate \openout \_prettytok_typeout_stream = {|\_prettytok_term_shell_decode_cmd}
	} {
		\int_const:Nn \_prettytok_typeout_stream {128}  % 16 works except on LuaLaTeX
	}


	\precattl_exec:n{
	\str_gset:Nn \_prettytok_gray {\cO\^^[[90m}
	\str_gset:Nn \_prettytok_red {\cO\^^[[31m}
	\str_gset:Nn \_prettytok_green {\cO\^^[[38;5;121m}  % this is the color vim use for green on my machine. Anyway~
	%\str_gset:Nn \_prettytok_green {\cO\^^[[92m}
	\str_gset:Nn \_prettytok_yellow {\cO\^^[[93m}
	%\str_gset:Nn \_prettytok_white {\cO\^^[[0m}  % i.e. the default
	\str_gset:Nn \_prettytok_white {\cO\^^[[97m}  % vim terminal has a quirk that if a line wraps, the status at the start of the line is considered "default" for that line, so need to explicitly specify color as white

	% https://stackoverflow.com/q/4842424/5267751

	\cs_new_protected:Npn \_prettytok_setcolor:N #1 {
		\ifx \_prettytok_current_color #1
		\else
			\let \_prettytok_current_color #1
			\tl_build_put_right:NV \_prettytok_content #1
		\fi
	}

	% caller should call \_prettytok_check_wrap: when useful.
	\cs_new_protected:Npn \_prettytok_term_prepare_print_char:n #1 {
		\_prettytok_append_content:x {
			\ifnum #1 < 27 ~
				\ifnum #1 = 9 ~  % tab
					\cO{⇥}
				\else
					\cO{\^\^} \char_generate:nn {#1+64} {12}
					\bool_if:nT {\int_compare_p:nNn {#1} = {13} || \int_compare_p:nNn {#1} = {10}} {
						\cO{^^J} \prettytermprefixmore
					}
				\fi
			\else
				\ifnum #1 = 32 ~
					\cO{␣}
				\else
					\bool_if:nTF {\int_compare_p:nNn {#1} > {32} && \int_compare_p:nNn {#1} < {127}} {  % printable ASCII character
						\char_generate:nn {#1} {12}
					} {
						\ifnum #1 < "100 ~
							^ ^
							\expandafter \@gobble \exp:w \exp_end_continue_f:w \int_to_hex:n {"100+#1}  % convert to hex, exactly 2 digits (by adding 100 then delete first '1')
						\else
							\ifnum #1 < "10000 ~
								^ ^ ^ ^
								\expandafter \@gobble \exp:w \exp_end_continue_f:w \int_to_hex:n {"10000+#1}  % convert to hex, exactly 4 digits
							\else
								^ ^ ^ ^ ^ ^
								\expandafter \@gobble \exp:w \exp_end_continue_f:w \int_to_hex:n {"1000000+#1}  % convert to hex, exactly 6 digits
							\fi
						\fi
					}
				\fi
			\fi
		}
	}
	\cs_generate_variant:Nn \_prettytok_term_prepare_print_char:n {x}

	\int_new:N \_prettytok_content_len

	% only call this on content that takes visible width. For example don't use this to set color
	\cs_new_protected:Npn \_prettytok_append_content:n #1 {
		\tl_build_put_right:Nn \_prettytok_content {#1}
		\int_add:Nn \_prettytok_content_len {\str_count:n {#1}}
	}

	\cs_generate_variant:Nn \_prettytok_append_content:n {V,x}

	\cs_new_protected:Npn \_prettytok_check_wrap: {
		\int_compare:nNnT {\_prettytok_content_len} > {\prettytermwraplimit} {
			% be careful to not color the \prettytermprefixmore part
			\ifx \_prettytok_current_color \_prettytok_white
			\else
				\tl_build_put_right:NV \_prettytok_content \_prettytok_white
			\fi
			\tl_build_end:N \_prettytok_content
			\_prettytok_typeout_content:

			\tl_build_begin:N \_prettytok_content
			\int_zero:N \_prettytok_content_len
			\_prettytok_append_content:V \prettytermprefixmore
			\ifx \_prettytok_current_color \_prettytok_white
			\else
				\tl_build_put_right:NV \_prettytok_content \_prettytok_current_color
			\fi
		}
	}

	% \_prettytok_content must have been called \tl_build_end:N on before this function is called
	\sys_if_engine_luatex:TF {
		\cs_new_protected:Npn \_prettytok_typeout_content: {
			\directlua{
				texio.write_nl("") 
				print(token.get_macro("_prettytok_content"))
			} % https://tex.stackexchange.com/questions/64462/lualatex-print-directlua-env-and-newlinehttps://tex.stackexchange.com/questions/64462/lualatex-print-directlua-env-and-newline
		}
	} {
		\cs_new_protected:Npn \_prettytok_typeout_content: {
			\immediate\write\_prettytok_typeout_stream{\_prettytok_content}
		}
	}

	\cs_new_protected:Npn \pretty:n #1 {
		\tl_build_begin:N \_prettytok_content
		\int_zero:N \_prettytok_content_len
		\_prettytok_append_content:V \prettytermprefix

		\let \_prettytok_current_color \_prettytok_white

		\tl_analysis_map_inline:nn {#1} {
			% by documentation of \tl_analysis_map_inline:nn: #1=token, #2=char code, #3=catcode
			\int_compare:nNnTF {##2} = {-1} {
				% token is control sequence
				\tl_if_eq:onTF {##1} {\cFrozenRelax} {
					\_prettytok_setcolor:N \_prettytok_red
					\_prettytok_append_content:n { \cO\\relax~ }
				} {
					\_prettytok_setcolor:N \_prettytok_yellow
					\_prettytok_append_content:n { \cO\\ }
					\tl_if_eq:xoF {\use:c {}} {##1} {
						\str_map_inline:xn {\exp_after:wN \cs_to_str:N ##1} {  % side note, must use str here to preserve spaces
							\_prettytok_term_prepare_print_char:x {\int_eval:n {`####1}}
						}
					}
					\_prettytok_append_content:n {~}
					\_prettytok_check_wrap:  % here we do not wrap in the middle of a csname
				}
			}
			{
				% token is not control sequence
				\if ##3 B   % letter
					\_prettytok_setcolor:N \_prettytok_green
				\else
					\if ##3 A   % spacer
						\_prettytok_setcolor:N \_prettytok_gray
					\else
						\if ##3 D  % active
							\_prettytok_setcolor:N \_prettytok_yellow
						\else
							\if ##3 C   % other cat
								\_prettytok_setcolor:N \_prettytok_white
							\else  % something else.
								\_prettytok_setcolor:N \_prettytok_red
							\fi
						\fi
					\fi
				\fi

				\_prettytok_term_prepare_print_char:n {##2}
				\_prettytok_check_wrap:
			}
		}
		\_prettytok_setcolor:N \_prettytok_white
		\tl_build_end:N \_prettytok_content

		\_prettytok_typeout_content:
	}
	}

}


% ======== Lua expandable variant.
\sys_if_engine_luatex:T {
	\cs_new:Npn \prettye:n #1 {
		\directlua{prettyprint(token.scan_toks())}{#1}
	}

	\cs_new:Npn \prettye:w {
		\directlua{prettyprintw()} {} {}
	}
	\cs_new:Npn \prettye:nw #1 {
		\directlua{prettyprintw()} {#1} {}
	}
	\cs_new:Npn \prettye:nnw #1 #2 {
		\directlua{prettyprintw()} {#1} {#2}
	}
}

% ======== w variant.

\cs_new_protected:Npn \pretty:w { \pretty_aux:w \empty }
\cs_new_protected:Npn \pretty_aux:w #1 \prettystop {
	\pretty:o { #1 }  % use o-expansion to remove the \empty
	#1 \prettystop
}
\cs_new:Npn \prettystop {}

% ======== other variants.
\cs_generate_variant:Nn \pretty:n {x, o, V}
\cs_new_eq:NN \pretty:N \pretty:n
\cs_generate_variant:Nn \pretty:N {c}

\cs_new_protected:Npn \prettythenrun:n #1 {
	\pretty:n {running~#1}
	#1
	\pretty:n {done}
}
\cs_new_protected:Npn \prettythenrun:w #1 \prettythenrunstop {
	\prettythenrun:n {#1}
}

\cs_new_protected:Nn \prettyshow:N { \pretty:o {\meaning #1} }
\cs_generate_variant:Nn \prettyshow:N {c}


% ======== generate \pretty:nn, \pretty:nnn, etc.
\begingroup
	\def \_prettytok_tmp:NN #1 #2 {   % #1: example \pretty:nn, #2: example \pretty:n (the latter has one more n than the former)
		\cs_new_protected:Npn #1 ##1 ##2 { #2 {##1 ##2} }
	}
	\cs_generate_variant:Nn \_prettytok_tmp:NN {cc}

	\def \_prettytok_tmp {pretty:n}
	\int_step_inline:nn {9} {
		\_prettytok_tmp:cc {\_prettytok_tmp n} {\_prettytok_tmp}
		\tl_put_right:Nn \_prettytok_tmp {n}
	}

	\sys_if_engine_luatex:T {
		\def \_prettytok_tmp:NN #1 #2 {   % #1: example \prettye:nn, #2: example \prettye:n (the latter has one more n than the former)
			\cs_new:Npn #1 ##1 ##2 { #2 {##1 ##2} }
		}
		\def \_prettytok_tmp {prettye:n}
		\int_step_inline:nn {9} {
			\_prettytok_tmp:cc {\_prettytok_tmp n} {\_prettytok_tmp}
			\tl_put_right:Nn \_prettytok_tmp {n}
		}

		\def \_prettytok_tmp:NN #1 #2 {   % #1: example \prettye:nnnw, #2: example \prettye:nnw (the latter has one more n than the former)
			\cs_new:Npn #1 ##1 ##2 ##3 { #2 {##1} {##2 ##3} }  % here ##1 is the callback (first ⟨n⟩), ##2 etc. are the additional-print things
		}
		\def \_prettytok_tmp {prettye:nn}
		\int_step_inline:nn {9} {
			\_prettytok_tmp:cc {\_prettytok_tmp nw} {\_prettytok_tmp w}
			\tl_put_right:Nn \_prettytok_tmp {n}
		}
	}
\endgroup

% ======== normal-catcode alias
\let\prettyN\pretty:n
\let\prettyX\pretty:x
\let\prettyO\pretty:o
\let\prettyV\pretty:V
\let\prettyW\pretty:w
\let\prettyeN\prettye:n
\let\prettyeW\prettye:w
\let\prettyshowN\prettyshow:N
\let\prettyshowC\prettyshow:c