Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

remove libdparse from token dump functionality #13

Open
wants to merge 13 commits into
base: replace_libdparse
Choose a base branch
from
Open

remove libdparse from token dump functionality #13

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3ef02b2
Update README
RazvanN7 Jan 10, 2022
19931fe
Add dmd-as-a-library submodule (#2)
RazvanN7 Jan 12, 2022
2b2f165
Add GH Actions build script (#4)
edi33416 Jan 26, 2022
31e0079
Removed libdparse from imports print functionality (#3)
lucica28 Jan 28, 2022
365464e
Fix failing tester + add unittest for imports (#7)
RazvanN7 Feb 2, 2022
a43dc46
Add style checker (#9)
edi33416 Feb 3, 2022
91fd756
Update action to build dlang fork
edi33416 Feb 10, 2022
6375c06
Fix linter errors
edi33416 Feb 10, 2022
8f95bc0
Add dmd dependencies to dub.json
edi33416 Feb 10, 2022
19b7912
Add dmd dependencies to build.bat
edi33416 Feb 10, 2022
e405313
remove libdparse from token dump functionality
lucica28 Mar 1, 2022
dc57664
use bytes previously computed and add output stream to api
lucica28 Mar 1, 2022
74c8f87
remove unnecessary file usage and make imports selective
lucica28 Mar 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/dscanner/main.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ else
else if (tokenDump)
{
string fileName = usingStdin ? "stdin" : args[1];
printTokenDump(fileName);
printTokenDump(stdout, bytes);
return 0;
}
}
Expand Down Expand Up @@ -276,7 +276,6 @@ else
LexerConfig config;
config.stringBehavior = StringBehavior.source;
auto tokens = byToken(readFile(f), config, &cache);

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unrelated change.

if (tokenCount)
count += printTokenCount(stdout, f, tokens);
else
Expand Down
80 changes: 34 additions & 46 deletions src/dscanner/stats.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,73 +53,61 @@ ulong printLineCount(Tokens)(File output, string fileName, ref Tokens tokens)
return count;
}

void printTokenDump(string fileName)
void printTokenDump(File output, ubyte[] bytes)
{
import dmd.tokens;
import dmd.lexer;
import std.file;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These imports need to be selective to pass the linter test.

import std.stdio : writefln;

scope lexer = new Lexer(null, cast(char*) bytes, 0, bytes.length, 0, 0);

auto input = readText(fileName);
scope lexer = new Lexer(null, input.ptr, 0, input.length, 0, 0);

lexer.nextToken;
Token t = lexer.token;

writeln("text \tblank\tindex\tline\tcolumn\ttype\tcomment\ttrailingComment");
output.writeln("text \tblank\tindex\tline\tcolumn\ttype\tcomment\ttrailingComment");
do
{
import std.stdio : writefln;
t = lexer.token;
string aux;
bool empty = true;
bool empty = false;

switch (t.value)
{
case TOK.string_:
aux = t.ustring[0 .. t.len].dup;
aux = "\"" ~ aux ~ "\"";
writef("<<%20s>>", aux);
empty = false;
output.writef("<<%20s>>", "\"" ~ t.ustring[0 .. t.len] ~ "\"");
break;
case TOK.int32Literal:
writef("<<%20d>>", t.intvalue);
empty = false;
output.writef("<<%20d>>", t.intvalue);
break;
case TOK.int64Literal:
writef("<<%20lldL>>", cast(long)t.intvalue);
empty = false;
output.writef("<<%20lldL>>", cast(long) t.intvalue);
break;
case TOK.uns32Literal:
writef("<<%20uU>>", t.unsvalue);
empty = false;
output.writef("<<%20uU>>", t.unsvalue);
break;
case TOK.uns64Literal:
writef("<<%20lluUL>>", t.unsvalue);
empty = false;
output.writef("<<%20lluUL>>", t.unsvalue);
break;
case TOK.float32Literal:
case TOK.float64Literal:
case TOK.float80Literal:
writef("<<%20g>>", t.floatvalue);
empty = false;
output.writef("<<%20g>>", t.floatvalue);
break;
case TOK.identifier:
writef("<<%20s>>", t.ident.toString());
empty = false;
output.writef("<<%20s>>", t.ident.toString());
break;
case TOK.wcharLiteral:
case TOK.dcharLiteral:
case TOK.charLiteral:
aux = "\'" ~ cast(char) t.unsvalue ~ "\'";
writef("<<%20s>>", aux);
empty = false;
output.writef("<<%20s>>", "\'" ~ cast(char) t.unsvalue ~ "\'");
break;
default:
writef("<<%20s>>", Token.toString(t.value));
empty = true;
output.writef("<<%20s>>", Token.toString(t.value));
break;
}

writefln("\t%b\t%d\t%d\t%d\t%d\t%s\t%s",
output.writefln("\t%b\t%d\t%d\t%d\t%d\t%s\t%s",
empty,
t.loc.fileOffset,
t.loc.linnum,
Expand All @@ -132,14 +120,17 @@ void printTokenDump(string fileName)

unittest
{
import std.file;
import core.stdc.stdio : freopen, stdout;
import core.sys.posix.unistd : dup, dup2;
import std.stdio : File;
import std.file : exists, remove;
import std.file : exists, remove, readText;
import dscanner.utils;

auto deleteme = "test.txt";
File file = File(deleteme, "w");
scope (exit)
{
assert(exists(deleteme));
remove(deleteme);
}

file.write(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we creating a file to put the text into it and then read that text back and pass it to the printTokenDump? We want to test only the functionality that we've added, which is the token dump. We can trust that the library implementation of read and write works correctly.

q{import std.stdio;
Expand All @@ -154,11 +145,16 @@ void main(string[] args)
file.close();

auto deleteme2 = "test2.txt";
auto fp = freopen("test2.txt", "w", stdout);
File file2 = File(deleteme2, "w");
scope (exit)
{
assert(exists(deleteme2));
remove(deleteme2);
}

printTokenDump(deleteme);
fflush(fp);
fclose(fp);
ubyte[] bytes = readFile(deleteme);
printTokenDump(file2, bytes);
file2.close();

auto actual = readText(deleteme2);
auto expected = "text blank index line column type comment trailingComment
Expand Down Expand Up @@ -194,13 +190,5 @@ void main(string[] args)
<< }>> 1 124 8 1 6
";

scope(exit)
{
assert(exists(deleteme));
remove(deleteme);
assert(exists(deleteme2));
remove(deleteme2);
}

assert(actual == expected);
}
}