diff --git a/src/main/java/link/locutus/discord/commands/manager/CommandManager.java b/src/main/java/link/locutus/discord/commands/manager/CommandManager.java index 9bad7bcc..fd04f458 100644 --- a/src/main/java/link/locutus/discord/commands/manager/CommandManager.java +++ b/src/main/java/link/locutus/discord/commands/manager/CommandManager.java @@ -306,7 +306,7 @@ public boolean run(Guild guild, IMessageIO channel, final User msgUser, String c } if (result != null && !result.isEmpty()) { result = result.replaceAll("(?i)" + Settings.INSTANCE.API_KEY_PRIMARY, "XXX"); - result = result.replaceAll("(?i)(?<=^|[^A-Fa-f0-9])(?:[0-9a-f]{2}){7,}(?=[^A-Fa-f0-9]|$)", "XXX"); +// result = result.replaceAll("(?i)(?<=^|[^A-Fa-f0-9])(?:[0-9a-f]{2}){7,}(?=[^A-Fa-f0-9]|$)", "XXX"); channel.send(result); } } catch (Throwable e) { diff --git a/src/main/java/link/locutus/discord/commands/manager/v2/impl/pw/commands/AdminCommands.java b/src/main/java/link/locutus/discord/commands/manager/v2/impl/pw/commands/AdminCommands.java index fc3f79a2..38e115e1 100644 --- a/src/main/java/link/locutus/discord/commands/manager/v2/impl/pw/commands/AdminCommands.java +++ b/src/main/java/link/locutus/discord/commands/manager/v2/impl/pw/commands/AdminCommands.java @@ -350,7 +350,8 @@ public String announce(@Me GuildDB db, @Me Guild guild, @Me JSONObject command, } } - List replacementLines = Arrays.asList(replacements.split("\n")); + List replacementLines = Arrays.asList(replacements.split("(? comments = updateAndGetNewComments(); + if (guild == null) return true; + List purgeCategories = new ArrayList<>(); for (DBComment comment : comments) { @@ -105,6 +113,113 @@ public void createTables() { e.printStackTrace(); } }; + { + String nations = "CREATE TABLE IF NOT EXISTS `FORUM_TOPICS` (`topic_id` INT PRIMARY KEY, `section_id` INT NOT NULL, `topic_name` VARCHAR NOT NULL, `topic_urlname` VARCHAR NOT NULL, `section_name` VARCHAR NOT NULL, `section_urlname` VARCHAR NOT NULL, `timestamp` BIGINT NOT NULL, `poster_id` INT NOT NULL, `poster_name` VARCHAR NOT NULL)"; + try (Statement stmt = getConnection().createStatement()) { + stmt.addBatch(nations); + stmt.executeBatch(); + stmt.clearBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + }; + } + + public void addTopic(DBTopic topic) throws SQLException { + String sql = "INSERT INTO `FORUM_TOPICS` (`topic_id`, `section_id`, `topic_name`, `topic_urlname`, `section_name`, `section_urlname`, `timestamp`, `poster_id`, `poster_name`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + try (PreparedStatement stmt = getConnection().prepareStatement(sql)) { + stmt.setInt(1, topic.topic_id); + stmt.setInt(2, topic.section_id); + stmt.setString(3, topic.topic_name); + stmt.setString(4, topic.topic_urlname); + stmt.setString(5, topic.section_name); + stmt.setString(6, topic.section_urlname); + stmt.setLong(7, topic.timestamp); + stmt.setInt(8, topic.poster_id); + stmt.setString(9, topic.poster_name); + stmt.execute(); + } + } + + private static String get(String requestURL) throws IOException { + URL website = new URL(requestURL); + URLConnection connection = website.openConnection(); + try (BufferedReader in = new BufferedReader( + new InputStreamReader(connection.getInputStream()))) { + + StringBuilder response = new StringBuilder(); + String inputLine; + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + return response.toString(); + } + } + + public void scrapeTopic(int section_id, String section_name) throws SQLException, IOException { + String baseUrl = "https://forum.politicsandwar.com/index.php?/forum/" + section_id + "-" + section_name + "/"; + int page = 1; + while (true) { + String url = baseUrl + "page/" + page + "/"; + + String content = get(url); + Document html = Jsoup.parse(content); + // get every ipsType_break ipsContained + Elements elems = html.select(".ipsType_break.ipsContained"); + for (Element elem : elems) { + // get href + Elements a = elem.select("a"); + String topicUrl = a.attr("href"); + String topicName = a.text(); + int topic_id = Integer.parseInt(topicUrl.split("topic/")[1].split("-")[0]); + + String topicUrlName = topicUrl.split("topic/")[1].split("-", 2)[1].split("/")[0]; + + // get date + Elements date = elem.select("time"); + String dateStr = date.attr("datetime"); + // to milliseconds + long timestamp = Instant.parse(dateStr).toEpochMilli(); + + // get poster id and name (611 and prefontaine) + Elements poster = elem.select("a[href^='https://forum.politicsandwar.com/index.php?/profile/']"); + String posterName = poster.text(); + String posterUrl = poster.attr("href"); + int posterId = Integer.parseInt(posterUrl.split("profile/")[1].split("-")[0]); + + // public DBTopic(int topic_id, int section_id, String topic_name, String topic_urlname, String section_name, String section_urlname, long timestamp, int poster_id, String poster_name) { + + System.out.println("Saving: " + topicName); + DBTopic topic = new DBTopic(topic_id, section_id, topicName, topicUrlName, section_name, section_name, timestamp, posterId, posterName); + addTopic(topic); + } + + // get highest data-page + Elements pages = html.select("li.ipsPagination_page"); + // if exists + if (pages.size() > 0) { + // get last page + Element lastPage = pages.get(pages.size() - 1); + // get href + String href = lastPage.select("a").attr("href"); + // get page number + int highestPage = Integer.parseInt(href.split("page/")[1].split("/")[0]); + if (highestPage > page) { + page++; + } else { + break; + } + } else { + break; + } + } + } + + public static void main(String[] args) throws SQLException, ClassNotFoundException, IOException { + ForumDB db = new ForumDB(null); + + db.scrapeTopic(42, "alliance-affairs"); } public List updateAndGetNewComments() throws IOException { diff --git a/src/main/java/link/locutus/discord/db/NationDB.java b/src/main/java/link/locutus/discord/db/NationDB.java index 1dbf741a..e0e1af1e 100644 --- a/src/main/java/link/locutus/discord/db/NationDB.java +++ b/src/main/java/link/locutus/discord/db/NationDB.java @@ -2958,17 +2958,19 @@ private void importLegacyNationLoot(boolean fromAttacks) throws SQLException { getDb().drop("NATION_LOOT"); } - Map> nationLoot = Locutus.imp().getWarDb().getNationLootFromAttacksLegacy(); - for (Map.Entry> entry : nationLoot.entrySet()) { - int nationId = entry.getKey(); - long date = entry.getValue().getKey(); - double[] loot = entry.getValue().getValue(); - NationLootType type = NationLootType.WAR_LOSS; - lootInfoList.add(new LootEntry(nationId, loot, date, type)); - } - - if (!lootInfoList.isEmpty()) { - saveNationLoot(lootInfoList); + if (fromAttacks) { + Map> nationLoot = Locutus.imp().getWarDb().getNationLootFromAttacksLegacy(); + for (Map.Entry> entry : nationLoot.entrySet()) { + int nationId = entry.getKey(); + long date = entry.getValue().getKey(); + double[] loot = entry.getValue().getValue(); + NationLootType type = NationLootType.WAR_LOSS; + lootInfoList.add(new LootEntry(nationId, loot, date, type)); + } + + if (!lootInfoList.isEmpty()) { + saveNationLoot(lootInfoList); + } } } diff --git a/src/main/java/link/locutus/discord/db/entities/DBTopic.java b/src/main/java/link/locutus/discord/db/entities/DBTopic.java new file mode 100644 index 00000000..1c915c4d --- /dev/null +++ b/src/main/java/link/locutus/discord/db/entities/DBTopic.java @@ -0,0 +1,26 @@ +package link.locutus.discord.db.entities; + +public class DBTopic { + public final int topic_id; + public final int section_id; + public final String topic_name; + public final String topic_urlname; + public final String section_name; + public final String section_urlname; + public final long timestamp; + public final int poster_id; + public final String poster_name; + + public DBTopic(int topic_id, int section_id, String topic_name, String topic_urlname, String section_name, String section_urlname, long timestamp, int poster_id, String poster_name) { + this.topic_id = topic_id; + this.section_id = section_id; + this.topic_name = topic_name; + this.topic_urlname = topic_urlname; + this.section_name = section_name; + this.section_urlname = section_urlname; + this.timestamp = timestamp; + this.poster_id = poster_id; + this.poster_name = poster_name; + } + +} diff --git a/src/main/java/link/locutus/discord/db/guild/GuildKey.java b/src/main/java/link/locutus/discord/db/guild/GuildKey.java index c0e50297..cd6ed012 100644 --- a/src/main/java/link/locutus/discord/db/guild/GuildKey.java +++ b/src/main/java/link/locutus/discord/db/guild/GuildKey.java @@ -115,53 +115,55 @@ public Set validate(GuildDB db, Set aaIds) { if (ownerNation == null || ownerNation.getAlliance_id() != aaId || ownerNation.getPosition() < Rank.LEADER.id) { Set inviteCodes = new HashSet<>(); boolean isValid = Roles.ADMIN.hasOnRoot(owner.getUser()); - try { + if (!isValid) { try { - List invites = RateLimitUtil.complete(db.getGuild().retrieveInvites()); - for (Invite invite : invites) { - String inviteCode = invite.getCode(); - inviteCodes.add(inviteCode); - } - } catch (InsufficientPermissionException ignore) { - } - - if (!inviteCodes.isEmpty() && alliance.getDiscord_link() != null && !alliance.getDiscord_link().isEmpty()) { - for (String code : inviteCodes) { - if (alliance.getDiscord_link().contains(code)) { - isValid = true; - break; + try { + List invites = RateLimitUtil.complete(db.getGuild().retrieveInvites()); + for (Invite invite : invites) { + String inviteCode = invite.getCode(); + inviteCodes.add(inviteCode); } + } catch (Throwable ignore) { } - } - - if (!isValid) { - String url = Settings.INSTANCE.PNW_URL() + "/alliance/id=" + aaId; - String content = FileUtil.readStringFromURL(PagePriority.ALLIANCE_ID_AUTH_CODE.ordinal(), url); - String idStr = db.getGuild().getId(); - if (!content.contains(idStr)) { - for (String inviteCode : inviteCodes) { - if (content.contains(inviteCode)) { + if (!inviteCodes.isEmpty() && alliance.getDiscord_link() != null && !alliance.getDiscord_link().isEmpty()) { + for (String code : inviteCodes) { + if (alliance.getDiscord_link().contains(code)) { isValid = true; break; } } - } else { - isValid = true; } - } - if (!isValid) { - String msg = "1. Go to: <" + Settings.INSTANCE.PNW_URL() + "/alliance/edit/id=" + aaId + ">\n" + - "2. Scroll down to where it says Alliance Description:\n" + - "3. Put your guild id `" + db.getIdLong() + "` somewhere in the text\n" + - "4. Click save\n" + - "5. Run the command " + getCommandObj(aaIds) + " again\n" + - "(note: you can remove the id after setup)"; - throw new IllegalArgumentException(msg); + if (!isValid) { + String url = Settings.INSTANCE.PNW_URL() + "/alliance/id=" + aaId; + String content = FileUtil.readStringFromURL(PagePriority.ALLIANCE_ID_AUTH_CODE.ordinal(), url); + String idStr = db.getGuild().getId(); + + if (!content.contains(idStr)) { + for (String inviteCode : inviteCodes) { + if (content.contains(inviteCode)) { + isValid = true; + break; + } + } + } else { + isValid = true; + } + } + + if (!isValid) { + String msg = "1. Go to: <" + Settings.INSTANCE.PNW_URL() + "/alliance/edit/id=" + aaId + ">\n" + + "2. Scroll down to where it says Alliance Description:\n" + + "3. Put your guild id `" + db.getIdLong() + "` somewhere in the text\n" + + "4. Click save\n" + + "5. Run the command " + getCommandObj(aaIds) + " again\n" + + "(note: you can remove the id after setup)"; + throw new IllegalArgumentException(msg); + } + } catch (IOException e) { + throw new RuntimeException(e); } - } catch (IOException e) { - throw new RuntimeException(e); } } diff --git a/src/main/java/link/locutus/discord/gpt/GPTUtil.java b/src/main/java/link/locutus/discord/gpt/GPTUtil.java index c3d84187..37cb3771 100644 --- a/src/main/java/link/locutus/discord/gpt/GPTUtil.java +++ b/src/main/java/link/locutus/discord/gpt/GPTUtil.java @@ -45,6 +45,7 @@ public static int getTokens(String input, ModelType type) { Encoding enc = registry.getEncodingForModel(type); return enc.encode(input).size(); } + public static List getChunks(String input, ModelType type, int tokenSizeCap) { List result = new ArrayList<>(); diff --git a/src/main/java/link/locutus/discord/gpt/GptHandler.java b/src/main/java/link/locutus/discord/gpt/GptHandler.java index b0d47608..fb137b1b 100644 --- a/src/main/java/link/locutus/discord/gpt/GptHandler.java +++ b/src/main/java/link/locutus/discord/gpt/GptHandler.java @@ -58,7 +58,7 @@ public GptHandler() throws SQLException, ClassNotFoundException { this.embeddingDatabase = new AdaEmbedding(registry, service); // TODO change ^ that to mini - File gpt4freePath = new File("../gpt4free/gpt3_5_turbo.py"); + File gpt4freePath = new File("../gpt4free/my_project/gpt3_5_turbo.py"); File venvExe = new File("../gpt4free/venv/Scripts/python.exe"); // ensure files exist if (!gpt4freePath.exists()) { diff --git a/src/main/java/link/locutus/discord/util/StringMan.java b/src/main/java/link/locutus/discord/util/StringMan.java index 3a358e79..3233a1c5 100644 --- a/src/main/java/link/locutus/discord/util/StringMan.java +++ b/src/main/java/link/locutus/discord/util/StringMan.java @@ -182,7 +182,7 @@ public static Set enumerateReplacements(String announcement, List