From 382940b73229520850d2ddde653347aea8947a51 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 4 Sep 2024 11:26:56 +1000 Subject: [PATCH] Prefer update all --- .../java/link/locutus/discord/Locutus.java | 15 ++-- .../link/locutus/discord/config/Settings.java | 17 ++-- .../link/locutus/discord/db/NationDB.java | 87 +++++++++---------- 3 files changed, 57 insertions(+), 62 deletions(-) diff --git a/src/main/java/link/locutus/discord/Locutus.java b/src/main/java/link/locutus/discord/Locutus.java index bdd61ac8..3eeb6f03 100644 --- a/src/main/java/link/locutus/discord/Locutus.java +++ b/src/main/java/link/locutus/discord/Locutus.java @@ -545,11 +545,11 @@ public void runUnsafe() throws Exception { taskTrack.addTask("Update Nations V2", () -> { runEventsAsync(events -> getNationDB().updateNationsV2(true, events)); - }, Settings.INSTANCE.TASKS.ALL_NON_VM_NATIONS_SECONDS, TimeUnit.SECONDS); + }, Settings.INSTANCE.TASKS.ALL_NATIONS_SECONDS, TimeUnit.SECONDS); taskTrack.addTask("City (V2)", () -> { runEventsAsync(events -> getNationDB().updateCitiesV2(events)); - }, Settings.INSTANCE.TASKS.ALL_NON_VM_NATIONS_SECONDS, TimeUnit.SECONDS); + }, Settings.INSTANCE.TASKS.ALL_CITIES_SECONDS, TimeUnit.SECONDS); taskTrack.addTask("War/Attack (V2)", () -> { synchronized (warUpdateLock) @@ -582,12 +582,16 @@ public void runUnsafe() throws Exception { taskTrack.addTask("Nation (Non VM)", () -> { runEventsAsync(events -> getNationDB().updateNonVMNations(events)); - }, Settings.INSTANCE.TASKS.ALL_NON_VM_NATIONS_SECONDS, TimeUnit.SECONDS); + }, Settings.INSTANCE.TASKS.ALL_NATIONS_SECONDS, TimeUnit.SECONDS); taskTrack.addTask("Outdated Cities", () -> { runEventsAsync(f -> getNationDB().updateDirtyCities(false, f)); }, Settings.INSTANCE.TASKS.OUTDATED_CITIES_SECONDS, TimeUnit.SECONDS); + taskTrack.addTask("Outdated Cities", () -> { + runEventsAsync(f -> getNationDB().updateAllCities(f)); + }, Settings.INSTANCE.TASKS.ALL_CITIES_SECONDS, TimeUnit.SECONDS); + if (Settings.INSTANCE.TASKS.FETCH_SPIES_INTERVAL_SECONDS > 0) { SpyUpdater spyUpdate = new SpyUpdater(); taskTrack.addTask("Spy Tracker", () -> { @@ -676,10 +680,7 @@ private void runTurnTasks(long lastTurn, long currentTurn) { // Update all nations { - runEventsAsync(events -> getNationDB().updateNonVMNations(events)); - } - { - runEventsAsync(events -> getNationDB().updateMostActiveNations(490, events)); + runEventsAsync(events -> getNationDB().updateAllNations(events, false)); } { runEventsAsync(events -> getNationDB().updateAlliances(null, events)); diff --git a/src/main/java/link/locutus/discord/config/Settings.java b/src/main/java/link/locutus/discord/config/Settings.java index bc5ae633..fc62a6be 100644 --- a/src/main/java/link/locutus/discord/config/Settings.java +++ b/src/main/java/link/locutus/discord/config/Settings.java @@ -204,17 +204,20 @@ public static class TASKS { // "Requires ALL_WAR_SECONDS to be enabled"}) // public boolean ESCALATION_ALERTS = true; - @Comment("Fetches most active nations (default 1 minute)") - public int ACTIVE_NATION_SECONDS = 60; + @Comment("Fetches most active nations (default disabled)") + public int ACTIVE_NATION_SECONDS = -1; - @Comment("Fetches colored nations (default 5 minutes)") - public int COLORED_NATIONS_SECONDS = 60 * 5; + @Comment("Fetches colored nations (default disabled)") + public int COLORED_NATIONS_SECONDS = -1; @Comment("Fetches non Vacation Mode nations (default 15 minutes)") - public int ALL_NON_VM_NATIONS_SECONDS = 60 * 15; + public int ALL_NATIONS_SECONDS = 60 * 15; - @Comment("Fetches outdated cities (default 5 minute)") - public int OUTDATED_CITIES_SECONDS = 60 * 5; + @Comment("Fetches outdated cities (default disabled)") + public int OUTDATED_CITIES_SECONDS = -1; + + @Comment("Fetches outdated cities (default 30 minute)") + public int ALL_CITIES_SECONDS = 60 * 30; @Comment("Runs the pre update beige reminders (default: 61 seconds)") public int BEIGE_REMINDER_SECONDS = 61; diff --git a/src/main/java/link/locutus/discord/db/NationDB.java b/src/main/java/link/locutus/discord/db/NationDB.java index c4f8f475..34f89a5f 100644 --- a/src/main/java/link/locutus/discord/db/NationDB.java +++ b/src/main/java/link/locutus/discord/db/NationDB.java @@ -73,6 +73,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -89,6 +90,7 @@ public class NationDB extends DBMainV2 implements SyncableDatabase { private final Map> positionsByAllianceId = new Int2ObjectOpenHashMap<>(); private final Map> treatiesByAlliance = new Int2ObjectOpenHashMap<>(); private final Set dirtyCities = Collections.synchronizedSet(new LinkedHashSet<>()); + private final Set dirtyCityNations = Collections.synchronizedSet(new LinkedHashSet<>()); private final Set dirtyNations = Collections.synchronizedSet(new LinkedHashSet<>()); private final Map> treasuresByNation = new Int2ObjectOpenHashMap<>(); private final Map treasuresByName = new ConcurrentHashMap<>(); @@ -162,17 +164,17 @@ public NationDB load() throws SQLException { // Load missing data if (Settings.INSTANCE.ENABLED_COMPONENTS.REPEATING_TASKS) { - if (nationsById.isEmpty() && (Settings.INSTANCE.TASKS.ACTIVE_NATION_SECONDS > 0 || Settings.INSTANCE.TASKS.COLORED_NATIONS_SECONDS > 0 || Settings.INSTANCE.TASKS.ALL_NON_VM_NATIONS_SECONDS > 0)) { + if (nationsById.isEmpty() && (Settings.INSTANCE.TASKS.ACTIVE_NATION_SECONDS > 0 || Settings.INSTANCE.TASKS.COLORED_NATIONS_SECONDS > 0 || Settings.INSTANCE.TASKS.ALL_NATIONS_SECONDS > 0)) { Logg.text("No nations loaded, fetching all"); updateAllNations(null, false); Logg.text("Done fetching all nations"); } - if (alliancesById.isEmpty() && (Settings.INSTANCE.TASKS.ACTIVE_NATION_SECONDS > 0 || Settings.INSTANCE.TASKS.COLORED_NATIONS_SECONDS > 0 || Settings.INSTANCE.TASKS.ALL_NON_VM_NATIONS_SECONDS > 0)) { + if (alliancesById.isEmpty() && (Settings.INSTANCE.TASKS.ACTIVE_NATION_SECONDS > 0 || Settings.INSTANCE.TASKS.COLORED_NATIONS_SECONDS > 0 || Settings.INSTANCE.TASKS.ALL_NATIONS_SECONDS > 0)) { Logg.text("No alliances loaded, fetching all"); updateAlliances(null, null); Logg.text("Done fetching all alliances"); } - if (citiesByNation.isEmpty() && (Settings.INSTANCE.TASKS.OUTDATED_CITIES_SECONDS > 0)) { + if (citiesByNation.isEmpty() && (Settings.INSTANCE.TASKS.OUTDATED_CITIES_SECONDS > 0 || Settings.INSTANCE.TASKS.ALL_CITIES_SECONDS > 0)) { Logg.text("No cities loaded, fetching all"); updateAllCities(null); Logg.text("Done fetching all cities"); @@ -1074,6 +1076,8 @@ private List getMostActiveNationIdsLimitCities(int numCitiesToFetch, Se } public void updateCitiesV2(Consumer eventConsumer) { + dirtyCities.clear(); + dirtyCityNations.clear(); Map citiesToDeleteToNationId = new HashMap<>(); synchronized (citiesByNation) { for (Map.Entry natEntry : citiesByNation.entrySet()) { @@ -1154,6 +1158,8 @@ public void updateCitiesV2(Consumer eventConsumer) { public void updateAllCities(Consumer eventConsumer) { PoliticsAndWarV3 v3 = Locutus.imp().getV3(); + dirtyCities.clear(); + dirtyCityNations.clear(); List cities = v3.readSnapshot(PagePriority.API_CITIES_AUTO_ALL, City.class); Map> nationIdCityIdCityMap = new Int2ObjectOpenHashMap<>(); for (City city : cities) { @@ -1163,19 +1169,26 @@ public void updateAllCities(Consumer eventConsumer) { updateCities(nationIdCityIdCityMap, eventConsumer); } - public void updateCitiesOfNations(Set nationIds, boolean priority, boolean bulk, Consumer eventConsumer) { - PoliticsAndWarV3 v3 = Locutus.imp().getV3(); - - List idList = new ArrayList<>(nationIds); + private int estimateCities(Set nationIds) { int estimatedCitiesToFetch = 0; - for (int nationId : idList) { + for (int nationId : nationIds) { DBNation nation = getNation(nationId); if (nation == null) estimatedCitiesToFetch++; else estimatedCitiesToFetch += nation.getCities(); } + return estimatedCitiesToFetch; + } + + public void updateCitiesOfNations(Set nationIds, boolean priority, boolean bulk, Consumer eventConsumer) { + PoliticsAndWarV3 v3 = Locutus.imp().getV3(); + List idList = new ArrayList<>(nationIds); + int estimatedCitiesToFetch = estimateCities(nationIds); + if (estimatedCitiesToFetch > 500) { + updateAllCities(eventConsumer); + return; + } if (bulk) { - // Pad with most outdated cities, to make full use of api call if (estimatedCitiesToFetch < 490) { // Slightly below 500 to avoid off by 1 errors with api int numToFetch = 490 - idList.size(); List mostActiveNations = getMostActiveNationIdsLimitCities(numToFetch, new HashSet<>(idList)); @@ -1186,6 +1199,7 @@ public void updateCitiesOfNations(Set nationIds, boolean priority, bool for (int i = 0; i < 500 && !idList.isEmpty(); i += 500) { int end = Math.min(i + 500, idList.size()); List toFetch = idList.subList(i, end); + toFetch.forEach(dirtyCityNations::remove); List cities = v3.fetchCitiesWithInfo(priority, r -> r.setNation_id(toFetch), true); Map> completeCities = new Int2ObjectOpenHashMap<>(); for (City city : cities) { @@ -1201,6 +1215,12 @@ public boolean updateDirtyCities(boolean priority, Consumer eventConsumer PoliticsAndWarV3 v3 = Locutus.imp().getV3(); markDirtyIncorrectNations(true, true); + int dirtyNationCities = estimateCities(dirtyCityNations); + if (dirtyCities.size() > 500 || dirtyNationCities > 500) { + updateAllCities(eventConsumer); + return true; + } + while (!dirtyCities.isEmpty()) { try { Iterator iter = dirtyCities.iterator(); @@ -1234,6 +1254,11 @@ public boolean updateDirtyCities(boolean priority, Consumer eventConsumer if (!deletedCities.isEmpty()) { deleteCities(deletedCities, eventConsumer); } + + if (!dirtyCityNations.isEmpty()) { + updateCitiesOfNations(dirtyCityNations, priority, false, eventConsumer); + } + return true; } @@ -1300,6 +1325,10 @@ private void updateCities(Map> completeCitiesByNatio synchronized (citiesByNation) { Object map = citiesByNation.get(nationId); if (map != null) { + int existingCities = ArrayUtil.countElements(DBCity.class, map); + if (existingCities == cities.size()) { + dirtyCityNations.remove(nationId); + } if (deleteMissing) { IntList toDelete = new IntArrayList(); ArrayUtil.iterateElements(DBCity.class, map, dbCity -> { @@ -1732,10 +1761,6 @@ public void updateNationsV2(boolean includeVM, Consumer eventConsumer) { if (!toSave.isEmpty()) saveNations(toSave); - if (!dirtyNationCities.isEmpty()) { - updateCitiesOfNations(dirtyNationCities, false, true, eventConsumer); - } - if (!expected.isEmpty()) { dirtyNations.addAll(expected); } @@ -1963,11 +1988,8 @@ public Set updateNationsCustom(Consumer> fetchNationT public void updateNationCitiesAndPositions(Collection allNations, Map nationChanges, Consumer eventConsumer) { boolean fetchCitiesIfNew = true; - boolean fetchCitiesIfOutdated = true; boolean fetchAlliancesIfOutdated = true; boolean fetchPositionsIfOutdated = true; - boolean fetchMostActiveIfNoneOutdated = false; - Set fetchCitiesOfNations = new HashSet<>(); if (!nationChanges.isEmpty()) { Set nationsToSave = new LinkedHashSet<>(); @@ -1979,48 +2001,17 @@ public void updateNationCitiesAndPositions(Collection allNations, Map< saveNations(nationsToSave); if (fetchCitiesIfNew) { - // num cities has changed for (Map.Entry entry : nationChanges.entrySet()) { DBNation prev = entry.getValue(); DBNation curr = entry.getKey(); if (curr == null) continue; if (prev == null || curr.getCities() != prev.getCities()) { - fetchCitiesOfNations.add(curr.getNation_id()); + dirtyCityNations.add(curr.getNation_id()); } } } } - if (fetchCitiesIfOutdated) { -// for (Map.Entry entry : nationChanges.entrySet()) { -// DBNation prev = entry.getValue(); -// DBNation curr = entry.getKey(); -// if (curr == null) continue; -// if (prev == null || (Math.round(100 * (curr.estimateScore() - curr.getScore())) != 0)) { -// if (Math.round((curr.getScore() - prev.getScore()) * 100) == 0) { -// System.out.println("Score change " + curr.getNation() + " " + curr.getScore() + " " + prev.getScore() + " " + curr.estimateScore()); -// } -// fetchCitiesOfNations.add(curr.getNation_id()); -// } -// } - for (DBNation nation : allNations) { - if (Math.round(100 * (nation.getScore() - PW.estimateScore(this, nation))) != 0) { - fetchCitiesOfNations.add(nation.getId()); - } - } - } - - if (dirtyNations.size() > 1000) { - Logg.text("Updated nations.\n" + - "Changes: " + nationChanges.size() + "\n" + - "Cities: " + fetchCitiesOfNations.size() + "\n" + - "Outdated: " + dirtyNations.size()); - } - if (!fetchCitiesOfNations.isEmpty() || (fetchMostActiveIfNoneOutdated)) { - updateCitiesOfNations(fetchCitiesOfNations, false, true, eventConsumer); - - } - if (fetchAlliancesIfOutdated || fetchPositionsIfOutdated) { updateOutdatedAlliances(fetchPositionsIfOutdated, eventConsumer); }