dfmt.tokens source code

1 //          Copyright Brian Schott 2015.
2 // Distributed under the Boost Software License, Version 1.0.
3 //    (See accompanying file LICENSE.txt or copy at
4 //          http://www.boost.org/LICENSE_1_0.txt)
5 
6 module dfmt.tokens;
7 
8 import dparse.lexer;
9 
10 /// Length of an invalid token
11 enum int INVALID_TOKEN_LENGTH = -1;
12 
13 uint betweenParenLength(const Token[] tokens) pure @safe @nogc
14 in
15 {
16     assert(tokens[0].type == tok!"(");
17 }
18 body
19 {
20     uint length = 0;
21     size_t i = 1;
22     int depth = 1;
23     while (i < tokens.length && depth > 0)
24     {
25         if (tokens[i].type == tok!"(")
26             depth++;
27         else if (tokens[i].type == tok!")")
28             depth--;
29         length += tokenLength(tokens[i]);
30         i++;
31     }
32     return length;
33 }
34 
35 int tokenLength(ref const Token t) pure @safe @nogc
36 {
37     import std.algorithm : countUntil;
38 
39     int c;
40     switch (t.type)
41     {
42     case tok!"doubleLiteral":
43     case tok!"floatLiteral":
44     case tok!"idoubleLiteral":
45     case tok!"ifloatLiteral":
46     case tok!"intLiteral":
47     case tok!"longLiteral":
48     case tok!"realLiteral":
49     case tok!"irealLiteral":
50     case tok!"uintLiteral":
51     case tok!"ulongLiteral":
52     case tok!"characterLiteral":
53         return cast(int) t.text.length;
54     case tok!"identifier":
55     case tok!"stringLiteral":
56     case tok!"wstringLiteral":
57     case tok!"dstringLiteral":
58         // TODO: Unicode line breaks and old-Mac line endings
59         c = cast(int) t.text.countUntil('\n');
60         if (c == -1)
61             return cast(int) t.text.length;
62         else
63             return c;
64         mixin(generateFixedLengthCases());
65     default:
66         return INVALID_TOKEN_LENGTH;
67     }
68 }
69 
70 bool isBreakToken(IdType t) pure nothrow @safe @nogc
71 {
72     switch (t)
73     {
74     case tok!"||":
75     case tok!"&&":
76     case tok!"(":
77     case tok!"[":
78     case tok!",":
79     case tok!":":
80     case tok!";":
81     case tok!"^^":
82     case tok!"^=":
83     case tok!"^":
84     case tok!"~=":
85     case tok!"<<=":
86     case tok!"<<":
87     case tok!"<=":
88     case tok!"<>=":
89     case tok!"<>":
90     case tok!"<":
91     case tok!"==":
92     case tok!"=>":
93     case tok!"=":
94     case tok!">=":
95     case tok!">>=":
96     case tok!">>>=":
97     case tok!">>>":
98     case tok!">>":
99     case tok!">":
100     case tok!"|=":
101     case tok!"|":
102     case tok!"-=":
103     case tok!"!<=":
104     case tok!"!<>=":
105     case tok!"!<>":
106     case tok!"!<":
107     case tok!"!=":
108     case tok!"!>=":
109     case tok!"!>":
110     case tok!"?":
111     case tok!"/=":
112     case tok!"/":
113     case tok!"..":
114     case tok!"*=":
115     case tok!"*":
116     case tok!"&=":
117     case tok!"%=":
118     case tok!"%":
119     case tok!"+=":
120     case tok!".":
121     case tok!"~":
122     case tok!"+":
123     case tok!"-":
124         return true;
125     default:
126         return false;
127     }
128 }
129 
130 int breakCost(IdType p, IdType c) pure nothrow @safe @nogc
131 {
132     switch (c)
133     {
134     case tok!"||":
135     case tok!"&&":
136     case tok!",":
137     case tok!":":
138     case tok!"?":
139         return 0;
140     case tok!"(":
141         return 60;
142     case tok!"[":
143         return 300;
144     case tok!";":
145     case tok!"^^":
146     case tok!"^=":
147     case tok!"^":
148     case tok!"~=":
149     case tok!"<<=":
150     case tok!"<<":
151     case tok!"<=":
152     case tok!"<>=":
153     case tok!"<>":
154     case tok!"<":
155     case tok!"==":
156     case tok!"=>":
157     case tok!"=":
158     case tok!">=":
159     case tok!">>=":
160     case tok!">>>=":
161     case tok!">>>":
162     case tok!">>":
163     case tok!">":
164     case tok!"|=":
165     case tok!"|":
166     case tok!"-=":
167     case tok!"!<=":
168     case tok!"!<>=":
169     case tok!"!<>":
170     case tok!"!<":
171     case tok!"!=":
172     case tok!"!>=":
173     case tok!"!>":
174     case tok!"/=":
175     case tok!"/":
176     case tok!"..":
177     case tok!"*=":
178     case tok!"*":
179     case tok!"&=":
180     case tok!"%=":
181     case tok!"%":
182     case tok!"+":
183     case tok!"-":
184     case tok!"~":
185     case tok!"+=":
186         return 200;
187     case tok!".":
188         return p == tok!")" ? 0 : 300;
189     default:
190         return 1000;
191     }
192 }
193 
194 pure nothrow @safe @nogc unittest
195 {
196     foreach (ubyte u; 0 .. ubyte.max)
197         if (isBreakToken(u))
198             assert(breakCost(tok!".", u) != 1000);
199 }
200 
201 private string generateFixedLengthCases()
202 {
203     import std.algorithm : map;
204     import std..string : format;
205     import std.array : join;
206 
207     assert(__ctfe);
208 
209     string[] spacedOperatorTokens = [
210         ",", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
211         "!>", "!>=", "%", "%=", "&", "&&", "&=", "*", "*=", "+", "+=", "-",
212         "-=", ":", ";", "<", "<<", "<<=", "<=", "<>", "<>=", "=", "==", "=>",
213         ">", ">=", ">>", ">>=", ">>>", ">>>=", "?", "@", "^", "^=", "^^",
214         "^^=", "|", "|=", "||", "~", "~="
215     ];
216     immutable spacedOperatorTokenCases = spacedOperatorTokens.map!(
217             a => format(`case tok!"%s": return %d + 1;`, a, a.length)).join("\n\t");
218 
219     string[] identifierTokens = [
220         "abstract", "alias", "align", "asm", "assert", "auto", "bool",
221         "break", "byte", "case", "cast", "catch", "cdouble", "cent", "cfloat", "char", "class",
222         "const", "continue", "creal", "dchar", "debug", "default", "delegate", "delete", "deprecated",
223         "do", "double", "else", "enum", "export", "extern", "false", "final", "finally", "float",
224         "for", "foreach", "foreach_reverse", "function", "goto", "idouble", "if", "ifloat", "immutable",
225         "import", "in", "inout", "int", "interface", "invariant", "ireal", "is",
226         "lazy", "long", "macro", "mixin", "module", "new", "nothrow", "null", "out", "override",
227         "package", "pragma", "private", "protected", "public", "pure", "real", "ref", "return", "scope",
228         "shared", "short", "static", "struct", "super", "switch", "synchronized", "template", "this",
229         "throw", "true", "try", "typedef", "typeid", "typeof", "ubyte", "ucent", "uint", "ulong",
230         "union", "unittest", "ushort", "version", "void", "volatile", "wchar",
231         "while", "with", "__DATE__", "__EOF__", "__FILE__",
232         "__FUNCTION__", "__gshared", "__LINE__", "__MODULE__", "__parameters",
233         "__PRETTY_FUNCTION__", "__TIME__", "__TIMESTAMP__",
234         "__traits", "__vector", "__VENDOR__", "__VERSION__", "$", "++", "--",
235         ".", "[", "]", "(", ")", "{", "}"
236     ];
237     immutable identifierTokenCases = identifierTokens.map!(
238             a => format(`case tok!"%s": return %d;`, a, a.length)).join("\n\t");
239     return spacedOperatorTokenCases ~ identifierTokenCases;
240 }