Skip to content

Commit

Permalink
Merge branch 'FeKozma:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
Livila authored Feb 25, 2024
2 parents da8adc6 + 36a5316 commit ae6740a
Show file tree
Hide file tree
Showing 13 changed files with 147 additions and 116 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
.gradle
build/
local-config.properties
model/
models/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/
Expand Down
21 changes: 10 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ You can edit some configuration in `config.properties`.
</summary>

###### *config.properties*
```shell
```shell
LOGGING_LEVEL=INFO
# This is related to TEMP_FILE below.
TEMP_FILE_ENABLED=false
Expand All @@ -90,18 +90,17 @@ LLM_FILE=models/ggml-model-gpt4all-falcon-q4_0.bin
RUN_WITH_LLM=false
# Directory path to where the code is to be scanned.
PATH_TO_CODE=src/test/resources/
```
```
</details>

#### LOGGING_LEVEL
The different options are the following:

| | FATAL | ERROR | WARNING | INFO | DEBUG | TRACE |
|----------|:-----:|:-----:|:-------:|:----:|:-----:|:-----:|
| OFF/NONE | | | | | | |
| FATAL | X | | | | | |
| ERROR | X | X | | | | |
| WARNING | X | X | X | | | |
| INFO | X | X | X | X | | |
| DEBUG | X | X | X | X | X | |
| TRACE | X | X | X | X | X | X |
| | ERROR | WARNING | INFO | DEBUG | TRACE |
|-----------|:-----:|:-------:|:----:|:-----:|:-----:|
| OFF/NONE | | | | | |
| ERROR | X | | | | |
| WARNING | X | X | | | |
| INFO | X | X | X | | |
| DEBUG | X | X | X | X | |
| TRACE | X | X | X | X | X |
2 changes: 1 addition & 1 deletion build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ dependencies {

tasks.test {
useJUnitPlatform()
}
}
1 change: 1 addition & 0 deletions gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ zipStorePath=wrapper/dists

org.gradle.daemon=true
org.gradle.parallel=true
# org.gradle.configuration-cache=true # And to disable, use `gradle --no-configuration-cache build` instead.
7 changes: 3 additions & 4 deletions src/main/java/CodeCheck/CodeCheck.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,9 @@
public interface CodeCheck {
static void execute() throws Exception {
ConfigInterface.conf.loadConfig();
String PATH_TO_CODE = ConfigInterface.conf.getString("PATH_TO_CODE");
String PATH_TO_CODE = Util.checkIfHomePath(ConfigInterface.conf.getString("PATH_TO_CODE"));
File baseCodeDir = new File(PATH_TO_CODE);

// Basically creating the file... // TODO: Only create the file if it's needed and not when it's empty.
WriteObjectToFile writeToFile = new WriteObjectToFile();

CheckDirectory checkDirectory = new CheckDirectory();
ManyFunctions manyFunctions;
if (!baseCodeDir.isDirectory()) {
Expand All @@ -28,6 +25,8 @@ static void execute() throws Exception {

checkDirectory.startModel();

WriteObjectToFile writeToFile = new WriteObjectToFile();

for (int i = 0; i < manyFunctions.oneFunctions.size(); i++) {
OneFunction oneFunction = manyFunctions.oneFunctions.get(i);
List<OneFunction> laterOneFunctions = manyFunctions.oneFunctions.subList(i + 1, manyFunctions.oneFunctions.size());
Expand Down
45 changes: 18 additions & 27 deletions src/main/java/CodeCheck/ConfigInterface.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package CodeCheck;

import java.io.*;
import java.nio.file.FileSystems;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand All @@ -12,49 +11,38 @@ public interface ConfigInterface {

class Config {
enum LoggingLevel {
NONE(0),
N(NONE.lvl),
OFF(NONE.lvl),
O(OFF.lvl),
ERROR(1),
ERR(ERROR.lvl),
E(ERROR.lvl),
WARNING(2),
WARN(WARNING.lvl),
W(WARNING.lvl),
INFO(3),
I(INFO.lvl),
DEBUG(4),
D(DEBUG.lvl),
TRACE(5),
T(TRACE.lvl);

private final int lvl;
NONE(0), N(NONE.lvl), OFF(NONE.lvl), O(OFF.lvl),
ERROR(1), ERR(ERROR.lvl), E(ERROR.lvl),
WARNING(2), WARN(WARNING.lvl), W(WARNING.lvl),
INFO(3), I(INFO.lvl),
DEBUG(4), D(DEBUG.lvl),
TRACE(5), T(TRACE.lvl);

public boolean logOn(LoggingLevel lvl) {
return lvl.lvl <= this.lvl;
}

LoggingLevel(int i) {
lvl = i;
private final int lvl;

LoggingLevel(int lvl) {
this.lvl = lvl;
}
}

Properties appProps;
private Properties appProps;

public Config() {}

public void loadConfig() throws IOException {
this.appProps = new Properties();

String rootDirPath = FileSystems.getDefault().getPath("").toAbsolutePath().toString() + File.separator;
String rootDirPath = System.getProperty("user.dir") + File.separator;
String appConfigPath = rootDirPath + "config.properties";
File test_conf = new File(rootDirPath + "test-config.properties");

if (test_conf.exists() && countLines(test_conf) > 1) {
if (test_conf.isFile() && countLines(test_conf) > 1) {
appConfigPath = rootDirPath + "test-config.properties";

} else if (new File(rootDirPath + "local-config.properties").exists()) {
} else if (new File(rootDirPath + "local-config.properties").isFile()) {
appConfigPath = rootDirPath + "local-config.properties";
}

Expand All @@ -77,7 +65,10 @@ public Boolean getBoolean(String key) {

public List<String> getList(String key) {
String value = getString(key);
if (value == null) return Collections.emptyList();

if (value == null)
return Collections.emptyList();

return Arrays
.stream(value.substring(1, value.length() - 1).split(","))
.map(String::strip)
Expand Down
32 changes: 20 additions & 12 deletions src/main/java/CodeCheck/LLM.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,39 @@

import com.hexadevlabs.gpt4all.LLModel;

import java.nio.file.Path;

import java.nio.file.Files;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;


public class LLM {
// GPT4All facade classes

LLModel model;
LLModel.GenerationConfig config;
long startTime;
private LLModel model;
private LLModel.GenerationConfig config;
private long startTime;

public void initModel() {
startTime = System.currentTimeMillis();

runIfConfig(() -> {
java.nio.file.Path modelPath = java.nio.file.Path.of(ConfigInterface.conf.getString("LLM_FILE"));
if (!Files.exists(modelPath)) {
Path modelPath = Path.of(Util.checkIfHomePath(
ConfigInterface.conf.getString("LLM_FILE")));

if (Files.isDirectory(modelPath)) {
Log.error("LLM_FILE not configured correctly - it is a directory, but was expecting a model file. Please change it into a correct model filepath.");
throw new RuntimeException("'LLM_FILE' is a directory, model file not found. [Model path: %s]".formatted(modelPath));
} else if (!Files.isRegularFile(modelPath) ) {
Log.error("LLM_FILE expected to be configured.");
throw new RuntimeException("LLM_FILE not found.");
throw new RuntimeException("LLM_FILE not found. [Model path: %s]".formatted(modelPath));
}

model = new LLModel(modelPath);

model.setThreadCount(6);

Log.debug("Thread Count: " + model.threadCount());

config = LLModel.config()
Expand All @@ -38,22 +44,22 @@ public void initModel() {

return true;
});

}

public void cleanModel() {
runIfConfig(() -> {
try {
model.close();
} catch (Exception e) {
Log.error("LLM is not closing...");
throw new RuntimeException("LLM is not closing...", e);
String errorMessage = "LLM is not closing...";
Log.error(errorMessage);
throw new RuntimeException(errorMessage, e);
}
return true;
});

int[] formattedTime = Util.getFormattedDurationTime(startTime);
Log.log(String.format("This LLM run completed in %02d hours, %02d minutes, %02d seconds and %02d milliseconds.",
Log.log(String.format("LLM run completed in %02d hours, %02d minutes, %02d seconds and %02d milliseconds.",
formattedTime[0], formattedTime[1], formattedTime[2], formattedTime[3]));
}

Expand All @@ -63,7 +69,9 @@ public String getAnswer(String question) {

return runIfConfig(() -> {
String answer = model.chatCompletion(createMessage(question), config).choices.toString();

Log.debug("Answer: " + answer);

return answer;
}).orElse("LLM has not been configured.");
}
Expand Down
41 changes: 14 additions & 27 deletions src/main/java/CodeCheck/Log.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ enum Color {
BLUE { @Override public String toString() { return "\u001B[34m"; } },
PURPLE { @Override public String toString() { return "\u001B[35m"; } },
BLACK { @Override public String toString() { return "\u001B[36m"; } },
CYAN { @Override public String toString() { return "\u001B[37m"; } };
GRAY { @Override public String toString() { return "\u001B[37m"; } };
}

static void log(String msg) {
Expand Down Expand Up @@ -44,7 +44,7 @@ static void log(String msg, ConfigInterface.Config.LoggingLevel lvl) {
logReduced("%s [%s] %s".formatted(
LocalDateTime.now(),
lvl.name(),
msg + System.lineSeparator()),
msg),
lvl);
}
}
Expand All @@ -57,7 +57,17 @@ static void log(String msg, ConfigInterface.Config.LoggingLevel level, Object...
log(String.format(msg, formatting), level);
}

static void logReduced(String msg, ConfigInterface.Config.LoggingLevel level, boolean newLine) {
static void log(String msg, Color color) {
System.out.printf(
"%s%s [%s] %s%n",
color,
LocalDateTime.now(),
ConfigInterface.Config.LoggingLevel.INFO,
msg
);
}

static private void logReduced(String msg, ConfigInterface.Config.LoggingLevel level) {
Color colorLevel = switch (level) {
case NONE, N, OFF, O -> Color.NONE; // No change in color.
case ERROR, ERR, E -> Color.RED;
Expand All @@ -67,29 +77,6 @@ static void logReduced(String msg, ConfigInterface.Config.LoggingLevel level, bo
case TRACE, T -> Color.BLUE;
};

if (newLine) System.out.println(colorLevel + msg + Color.RESET);
else System.out.print(colorLevel + msg + Color.RESET);
}

static void logReduced(String msg, ConfigInterface.Config.LoggingLevel level) {
logReduced(msg, level, false);
}

static void logReduced(String msg) {
logReduced(msg, ConfigInterface.Config.LoggingLevel.INFO, false);
}

static void logReduced(String msg, Object... formatting) {
logReduced(String.format(msg, formatting), ConfigInterface.Config.LoggingLevel.INFO, false);
}

static void logColor(String msg, Color color) {
System.out.printf(
"%s%s [%s] %s%n",
color,
LocalDateTime.now(),
ConfigInterface.Config.LoggingLevel.INFO,
msg
);
System.out.println(colorLevel + msg + Color.RESET);
}
}
5 changes: 3 additions & 2 deletions src/main/java/CodeCheck/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

public class Main {
public static void main(String[] args) throws Exception {
Log.log("Starting up...", "%n");
Log.log("Starting up...");
Log.log("Running on %s.".formatted(System.getProperty("os.name")));

CodeCheck.execute();

Log.log("Shutting down...", "%n");
Log.log("Shutting down...");
}
}
15 changes: 12 additions & 3 deletions src/main/java/CodeCheck/Util.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,16 @@ public interface Util {

static File createFile(String filePath, boolean isFirstTry) {
File file = new File(filePath);
try {

try {
if (file.createNewFile()) {
Log.log(String.format("File created: %s", file.getPath()));
return file;
} else {
if (isFirstTry)
Log.logReduced("The file %s already exist, trying to create the next id... ", file.getPath());
Log.log("The file %s already exist, trying to create the next id... ", file.getPath());
else
Log.logReduced("%s ... ", file.getPath());
Log.log("%s ... ", file.getPath());

return null;
}
Expand All @@ -31,6 +31,7 @@ static File createFile(String filePath, boolean isFirstTry) {
static void write(String filePath, String msg) {
write(filePath, msg, true);
}

static void write(String filePath, String msg, boolean append) {
try {
FileWriter myWriter = new FileWriter(filePath, append);
Expand All @@ -56,4 +57,12 @@ static int[] getFormattedDurationTime(long startTime) {

return new int[]{hours, minutes, seconds % 60, (int) millis % 1000};
}

static String checkIfHomePath(String path) {
if (path.startsWith("~")) {
return System.getProperty("user.home") + path.substring(1);
} else {
return path;
}
}
}
Loading

0 comments on commit ae6740a

Please sign in to comment.