dfmt.tokens source code

1 //          Copyright Brian Schott 2015.
2 // Distributed under the Boost Software License, Version 1.0.
3 //    (See accompanying file LICENSE.txt or copy at
4 //          http://www.boost.org/LICENSE_1_0.txt)
5 
6 module dfmt.tokens;
7 
8 import dparse.lexer;
9 
10 /// Length of an invalid token
11 enum int INVALID_TOKEN_LENGTH = -1;
12 
13 uint betweenParenLength(const Token[] tokens) pure @safe @nogc
14 in
15 {
16     assert(tokens[0].type == tok!"(");
17 }
18 do
19 {
20     uint length = 0;
21     size_t i = 1;
22     int depth = 1;
23     while (i < tokens.length && depth > 0)
24     {
25         if (tokens[i].type == tok!"(")
26             depth++;
27         else if (tokens[i].type == tok!")")
28             depth--;
29         length += tokenLength(tokens[i]);
30         i++;
31     }
32     return length;
33 }
34 
35 int tokenLength(ref const Token t) pure @safe @nogc
36 {
37     import std.algorithm : countUntil;
38 
39     int c;
40     switch (t.type)
41     {
42     case tok!"doubleLiteral":
43     case tok!"floatLiteral":
44     case tok!"idoubleLiteral":
45     case tok!"ifloatLiteral":
46     case tok!"intLiteral":
47     case tok!"longLiteral":
48     case tok!"realLiteral":
49     case tok!"irealLiteral":
50     case tok!"uintLiteral":
51     case tok!"ulongLiteral":
52     case tok!"characterLiteral":
53         return cast(int) t.text.length;
54     case tok!"identifier":
55     case tok!"stringLiteral":
56     case tok!"wstringLiteral":
57     case tok!"dstringLiteral":
58         // TODO: Unicode line breaks and old-Mac line endings
59         c = cast(int) t.text.countUntil('\n');
60         if (c == -1)
61             return cast(int) t.text.length;
62         else
63             return c;
64         mixin(generateFixedLengthCases());
65     default:
66         return INVALID_TOKEN_LENGTH;
67     }
68 }
69 
70 bool isBreakToken(IdType t) pure nothrow @safe @nogc
71 {
72     switch (t)
73     {
74     case tok!"||":
75     case tok!"&&":
76     case tok!"(":
77     case tok!"[":
78     case tok!",":
79     case tok!":":
80     case tok!";":
81     case tok!"^^":
82     case tok!"^=":
83     case tok!"^":
84     case tok!"~=":
85     case tok!"<<=":
86     case tok!"<<":
87     case tok!"<=":
88     case tok!"<>=":
89     case tok!"<>":
90     case tok!"<":
91     case tok!"==":
92     case tok!"=>":
93     case tok!"=":
94     case tok!">=":
95     case tok!">>=":
96     case tok!">>>=":
97     case tok!">>>":
98     case tok!">>":
99     case tok!">":
100     case tok!"|=":
101     case tok!"|":
102     case tok!"-=":
103     case tok!"!<=":
104     case tok!"!<>=":
105     case tok!"!<>":
106     case tok!"!<":
107     case tok!"!=":
108     case tok!"!>=":
109     case tok!"!>":
110     case tok!"?":
111     case tok!"/=":
112     case tok!"/":
113     case tok!"..":
114     case tok!"*=":
115     case tok!"*":
116     case tok!"&=":
117     case tok!"%=":
118     case tok!"%":
119     case tok!"+=":
120     case tok!".":
121     case tok!"~":
122     case tok!"+":
123     case tok!"-":
124         return true;
125     default:
126         return false;
127     }
128 }
129 
130 int breakCost(IdType p, IdType c) pure nothrow @safe @nogc
131 {
132     switch (c)
133     {
134     case tok!"||":
135     case tok!"&&":
136     case tok!",":
137     case tok!"?":
138         return 0;
139     case tok!"(":
140         return 60;
141     case tok!"[":
142         return 300;
143     case tok!";":
144     case tok!"^^":
145     case tok!"^=":
146     case tok!"^":
147     case tok!"~=":
148     case tok!"<<=":
149     case tok!"<<":
150     case tok!"<=":
151     case tok!"<>=":
152     case tok!"<>":
153     case tok!"<":
154     case tok!"==":
155     case tok!"=>":
156     case tok!"=":
157     case tok!">=":
158     case tok!">>=":
159     case tok!">>>=":
160     case tok!">>>":
161     case tok!">>":
162     case tok!">":
163     case tok!"|=":
164     case tok!"|":
165     case tok!"-=":
166     case tok!"!<=":
167     case tok!"!<>=":
168     case tok!"!<>":
169     case tok!"!<":
170     case tok!"!=":
171     case tok!"!>=":
172     case tok!"!>":
173     case tok!"/=":
174     case tok!"/":
175     case tok!"..":
176     case tok!"*=":
177     case tok!"*":
178     case tok!"&=":
179     case tok!"%=":
180     case tok!"%":
181     case tok!"+":
182     case tok!"-":
183     case tok!"~":
184     case tok!"+=":
185         return 200;
186     case tok!":":
187         // colon could be after a label or an import, where it should normally wrap like before
188         // for everything else (associative arrays) try not breaking around colons
189         return p == tok!"identifier" ? 0 : 300;
190     case tok!".":
191         return p == tok!")" ? 0 : 300;
192     default:
193         return 1000;
194     }
195 }
196 
197 pure nothrow @safe @nogc unittest
198 {
199     foreach (ubyte u; 0 .. ubyte.max)
200         if (isBreakToken(u))
201             assert(breakCost(tok!".", u) != 1000);
202 }
203 
204 private string generateFixedLengthCases()
205 {
206     import std.algorithm : map;
207     import std..string : format;
208     import std.array : join;
209 
210     assert(__ctfe);
211 
212     string[] spacedOperatorTokens = [
213         ",", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
214         "!>", "!>=", "%", "%=", "&", "&&", "&=", "*", "*=", "+", "+=", "-",
215         "-=", ":", ";", "<", "<<", "<<=", "<=", "<>", "<>=", "=", "==", "=>",
216         ">", ">=", ">>", ">>=", ">>>", ">>>=", "?", "@", "^", "^=", "^^",
217         "^^=", "|", "|=", "||", "~", "~="
218     ];
219     immutable spacedOperatorTokenCases = spacedOperatorTokens.map!(
220             a => format(`case tok!"%s": return %d + 1;`, a, a.length)).join("\n\t");
221 
222     string[] identifierTokens = [
223         "abstract", "alias", "align", "asm", "assert", "auto", "bool",
224         "break", "byte", "case", "cast", "catch", "cdouble", "cent", "cfloat", "char", "class",
225         "const", "continue", "creal", "dchar", "debug", "default", "delegate", "delete", "deprecated",
226         "do", "double", "else", "enum", "export", "extern", "false", "final", "finally", "float",
227         "for", "foreach", "foreach_reverse", "function", "goto", "idouble", "if", "ifloat", "immutable",
228         "import", "in", "inout", "int", "interface", "invariant", "ireal", "is",
229         "lazy", "long", "macro", "mixin", "module", "new", "nothrow", "null", "out", "override",
230         "package", "pragma", "private", "protected", "public", "pure", "real", "ref", "return", "scope",
231         "shared", "short", "static", "struct", "super", "switch", "synchronized", "template", "this",
232         "throw", "true", "try", "typedef", "typeid", "typeof", "ubyte", "ucent", "uint", "ulong",
233         "union", "unittest", "ushort", "version", "void", "wchar",
234         "while", "with", "__DATE__", "__EOF__", "__FILE__",
235         "__FUNCTION__", "__gshared", "__LINE__", "__MODULE__", "__parameters",
236         "__PRETTY_FUNCTION__", "__TIME__", "__TIMESTAMP__",
237         "__traits", "__vector", "__VENDOR__", "__VERSION__", "$", "++", "--",
238         ".", "[", "]", "(", ")", "{", "}"
239     ];
240     immutable identifierTokenCases = identifierTokens.map!(
241             a => format(`case tok!"%s": return %d;`, a, a.length)).join("\n\t");
242     return spacedOperatorTokenCases ~ identifierTokenCases;
243 }