From 47a0db2fb11515c52c3559b301a8e8d611355670 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Mon, 26 Jun 2023 15:37:28 +0200 Subject: [PATCH 1/2] importer: add and log newImageFiles + maxLoyaltyCardId --- .../main/java/protect/card_locker/DBHelper.java | 8 ++++++++ .../importexport/CatimaImporter.java | 4 +++- .../card_locker/importexport/FidmeImporter.java | 5 +++-- .../card_locker/importexport/Importer.java | 3 ++- .../importexport/MultiFormatImporter.java | 12 +++++++++++- .../importexport/StocardImporter.java | 17 ++++++++++++----- .../importexport/VoucherVaultImporter.java | 5 +++-- 7 files changed, 42 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/protect/card_locker/DBHelper.java b/app/src/main/java/protect/card_locker/DBHelper.java index 53ec2ea5ca..3eddb0a9eb 100644 --- a/app/src/main/java/protect/card_locker/DBHelper.java +++ b/app/src/main/java/protect/card_locker/DBHelper.java @@ -364,6 +364,14 @@ private static void updateFTS(final SQLiteDatabase db, final int id, final Strin whereAttrs(LoyaltyCardDbFTS.ID), withArgs(id)); } + public static int getMaxLoyaltyCardId(final SQLiteDatabase database) { + Cursor data = database.rawQuery("SELECT IFNULL(MAX(" + LoyaltyCardDbIds.ID + "), 0) FROM " + LoyaltyCardDbIds.TABLE, null, null); + data.moveToFirst(); + int maxId = data.getInt(0); + data.close(); + return maxId; + } + public static long insertLoyaltyCard( final SQLiteDatabase database, final String store, final String note, final Date validFrom, final Date expiry, final BigDecimal balance, final Currency balanceType, final String cardId, diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 52d521781c..8200002e6f 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -23,6 +23,7 @@ import java.util.Currency; import java.util.Date; import java.util.List; +import java.util.Set; import protect.card_locker.CatimaBarcode; import protect.card_locker.DBHelper; @@ -39,7 +40,7 @@ * A header is expected for the each table showing the names of the columns. */ public class CatimaImporter implements Importer { - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, InterruptedException { + public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException { InputStream bufferedInputStream = new BufferedInputStream(input); bufferedInputStream.mark(100); @@ -57,6 +58,7 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp importCSV(context, database, zipInputStream); } else if (fileName.endsWith(".png")) { Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream), fileName); + newImageFiles.add(fileName); } else { throw new FormatException("Unexpected file in import: " + fileName); } diff --git a/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java b/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java index b1e411a0f1..4c9add8e32 100644 --- a/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java @@ -17,6 +17,7 @@ import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.text.ParseException; +import java.util.Set; import protect.card_locker.CatimaBarcode; import protect.card_locker.DBHelper; @@ -31,7 +32,7 @@ * A header is expected for the each table showing the names of the columns. */ public class FidmeImporter implements Importer { - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, JSONException, ParseException { + public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException { // We actually retrieve a .zip file ZipInputStream zipInputStream = new ZipInputStream(input, password); @@ -130,4 +131,4 @@ private void importLoyaltyCard(Context context, SQLiteDatabase database, CSVReco DBHelper.insertLoyaltyCard(database, store, note, null, null, BigDecimal.valueOf(0), null, cardId, null, barcodeType, headerColor, starStatus, null,archiveStatus); } -} \ No newline at end of file +} diff --git a/app/src/main/java/protect/card_locker/importexport/Importer.java b/app/src/main/java/protect/card_locker/importexport/Importer.java index 41f73df262..bb938d079e 100644 --- a/app/src/main/java/protect/card_locker/importexport/Importer.java +++ b/app/src/main/java/protect/card_locker/importexport/Importer.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.io.InputStream; import java.text.ParseException; +import java.util.Set; import protect.card_locker.FormatException; @@ -23,5 +24,5 @@ public interface Importer { * @throws IOException * @throws FormatException */ - void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, InterruptedException, JSONException, ParseException; + void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException, JSONException, ParseException; } diff --git a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java index a5306dd336..df7fe18a8d 100644 --- a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java @@ -7,6 +7,10 @@ import net.lingala.zip4j.exception.ZipException; import java.io.InputStream; +import java.util.HashSet; +import java.util.Set; + +import protect.card_locker.DBHelper; public class MultiFormatImporter { private static final String TAG = "Catima"; @@ -41,11 +45,17 @@ public static ImportExportResult importData(Context context, SQLiteDatabase data } String error = null; + Set newImageFiles = new HashSet<>(); if (importer != null) { database.beginTransaction(); try { - importer.importData(context, database, input, password); + int maxLoyaltyCardId = DBHelper.getMaxLoyaltyCardId(database); + Log.d(TAG, "Current max loyalty card id: " + maxLoyaltyCardId); + importer.importData(context, database, input, password, newImageFiles, maxLoyaltyCardId); database.setTransactionSuccessful(); + for (String fileName : newImageFiles) { + Log.d(TAG, "New image file: " + fileName); + } return new ImportExportResult(ImportExportResultType.Success); } catch (ZipException e) { if (e.getType().equals(ZipException.Type.WRONG_PASSWORD)) { diff --git a/app/src/main/java/protect/card_locker/importexport/StocardImporter.java b/app/src/main/java/protect/card_locker/importexport/StocardImporter.java index 1aa90e3e03..0f011255be 100644 --- a/app/src/main/java/protect/card_locker/importexport/StocardImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/StocardImporter.java @@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.HashMap; +import java.util.Set; import protect.card_locker.CatimaBarcode; import protect.card_locker.DBHelper; @@ -42,7 +43,7 @@ public class StocardImporter implements Importer { private static final String TAG = "Catima"; - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, JSONException, ParseException { + public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException { HashMap> loyaltyCardHashMap = new HashMap<>(); HashMap> providers = new HashMap<>(); @@ -233,14 +234,20 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp long loyaltyCardInternalId = DBHelper.insertLoyaltyCard(database, store, note, null, null, BigDecimal.valueOf(0), null, cardId, null, barcodeType, headerColor, 0, null,0); if (cardIcon != null) { - Utils.saveCardImage(context, cardIcon, (int) loyaltyCardInternalId, ImageLocationType.icon); + String fileName = Utils.getCardImageFileName((int) loyaltyCardInternalId, ImageLocationType.icon); + Utils.saveCardImage(context, cardIcon, fileName); + newImageFiles.add(fileName); } if (loyaltyCardData.containsKey("frontImage")) { - Utils.saveCardImage(context, (Bitmap) loyaltyCardData.get("frontImage"), (int) loyaltyCardInternalId, ImageLocationType.front); + String fileName = Utils.getCardImageFileName((int) loyaltyCardInternalId, ImageLocationType.front); + Utils.saveCardImage(context, (Bitmap) loyaltyCardData.get("frontImage"), fileName); + newImageFiles.add(fileName); } if (loyaltyCardData.containsKey("backImage")) { - Utils.saveCardImage(context, (Bitmap) loyaltyCardData.get("backImage"), (int) loyaltyCardInternalId, ImageLocationType.back); + String fileName = Utils.getCardImageFileName((int) loyaltyCardInternalId, ImageLocationType.back); + Utils.saveCardImage(context, (Bitmap) loyaltyCardData.get("backImage"), fileName); + newImageFiles.add(fileName); } } @@ -272,4 +279,4 @@ private HashMap> appendToHashMap(HashMap newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException { BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)); StringBuilder sb = new StringBuilder(); @@ -131,4 +132,4 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp bufferedReader.close(); } -} \ No newline at end of file +} From 6e4a944bd1878e9a2793ec8bdad141da15fa3315 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Mon, 26 Jun 2023 16:40:11 +0200 Subject: [PATCH 2/2] catima importer: renumber IDs to merge properly --- .../main/java/protect/card_locker/Utils.java | 19 +++++++ .../importexport/CatimaImporter.java | 54 +++++++++++-------- .../importexport/MultiFormatImporter.java | 4 ++ .../protect/card_locker/ImportExportTest.java | 22 ++++++-- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index d98439e66c..c538b3af75 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -58,6 +58,8 @@ import java.util.GregorianCalendar; import java.util.Locale; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import protect.card_locker.preferences.Settings; @@ -380,6 +382,23 @@ static public String getCardImageFileName(int loyaltyCardId, ImageLocationType t return cardImageFileNameBuilder.toString(); } + static public String getRenamedCardImageFileName(final String fileName, final long idOffset) { + Pattern pattern = Pattern.compile("^(card_)(\\d+)(_(?:front|back|icon)\\.png)$"); + Matcher matcher = pattern.matcher(fileName); + if (matcher.matches()) { + StringBuilder cardImageFileNameBuilder = new StringBuilder(); + cardImageFileNameBuilder.append(matcher.group(1)); + try { + cardImageFileNameBuilder.append(Integer.parseInt(matcher.group(2)) + idOffset); + } catch (NumberFormatException _e) { + return null; + } + cardImageFileNameBuilder.append(matcher.group(3)); + return cardImageFileNameBuilder.toString(); + } + return null; + } + static public void saveCardImage(Context context, Bitmap bitmap, String fileName) throws FileNotFoundException { if (bitmap == null) { context.deleteFile(fileName); diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 8200002e6f..863a01acfb 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -55,10 +55,14 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); if (fileName.equals("catima.csv")) { - importCSV(context, database, zipInputStream); + importCSV(context, database, zipInputStream, maxLoyaltyCardId); } else if (fileName.endsWith(".png")) { - Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream), fileName); - newImageFiles.add(fileName); + String newFileName = Utils.getRenamedCardImageFileName(fileName, maxLoyaltyCardId); + if (newFileName == null) { + throw new FormatException("Unexpected PNG file in import: " + fileName); + } + Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream), newFileName); + newImageFiles.add(newFileName); } else { throw new FormatException("Unexpected file in import: " + fileName); } @@ -67,34 +71,34 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp if (!isZipFile) { // This is not a zip file, try importing as bare CSV bufferedInputStream.reset(); - importCSV(context, database, bufferedInputStream); + importCSV(context, database, bufferedInputStream, maxLoyaltyCardId); } input.close(); } - public void importCSV(Context context, SQLiteDatabase database, InputStream input) throws IOException, FormatException, InterruptedException { + public void importCSV(Context context, SQLiteDatabase database, InputStream input, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException { BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)); int version = parseVersion(bufferedReader); switch (version) { case 1: - parseV1(database, bufferedReader); + parseV1(database, bufferedReader, maxLoyaltyCardId); break; case 2: - parseV2(context, database, bufferedReader); + parseV2(context, database, bufferedReader, maxLoyaltyCardId); break; default: throw new FormatException(String.format("No code to parse version %s", version)); } } - public void parseV1(SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException { + public void parseV1(SQLiteDatabase database, BufferedReader input, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException { final CSVParser parser = new CSVParser(input, CSVFormat.RFC4180.builder().setHeader().build()); try { for (CSVRecord record : parser) { - importLoyaltyCard(database, record); + importLoyaltyCard(database, record, maxLoyaltyCardId); if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); @@ -107,7 +111,7 @@ public void parseV1(SQLiteDatabase database, BufferedReader input) throws IOExce } } - public void parseV2(Context context, SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException { + public void parseV2(Context context, SQLiteDatabase database, BufferedReader input, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException { int part = 0; StringBuilder stringPart = new StringBuilder(); @@ -133,7 +137,7 @@ public void parseV2(Context context, SQLiteDatabase database, BufferedReader inp break; case 2: try { - parseV2Cards(context, database, stringPart.toString()); + parseV2Cards(context, database, stringPart.toString(), maxLoyaltyCardId); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -141,7 +145,7 @@ public void parseV2(Context context, SQLiteDatabase database, BufferedReader inp break; case 3: try { - parseV2CardGroups(database, stringPart.toString()); + parseV2CardGroups(database, stringPart.toString(), maxLoyaltyCardId); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -195,7 +199,7 @@ public void parseV2Groups(SQLiteDatabase database, String data) throws IOExcepti } } - public void parseV2Cards(Context context, SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public void parseV2Cards(Context context, SQLiteDatabase database, String data, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException { // Parse cards final CSVParser cardParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -216,11 +220,11 @@ public void parseV2Cards(Context context, SQLiteDatabase database, String data) } for (CSVRecord record : records) { - importLoyaltyCard(database, record); + importLoyaltyCard(database, record, maxLoyaltyCardId); } } - public void parseV2CardGroups(SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public void parseV2CardGroups(SQLiteDatabase database, String data, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException { // Parse card group mappings final CSVParser cardGroupParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -241,7 +245,7 @@ public void parseV2CardGroups(SQLiteDatabase database, String data) throws IOExc } for (CSVRecord record : records) { - importCardGroupMapping(database, record); + importCardGroupMapping(database, record, maxLoyaltyCardId); } } @@ -278,9 +282,12 @@ private int parseVersion(BufferedReader reader) throws IOException { * Import a single loyalty card into the database using the given * session. */ - private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record) + private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record, int maxLoyaltyCardId) throws FormatException { int id = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIds.ID, record); + if (id < 1) { + throw new FormatException("ID must be >= 1"); + } String store = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIds.STORE, record, ""); if (store.isEmpty()) { @@ -376,7 +383,8 @@ private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record) // We catch this exception so we can still import old backups } - DBHelper.insertLoyaltyCard(database, id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, archiveStatus); + int newId = id + maxLoyaltyCardId; + DBHelper.insertLoyaltyCard(database, newId, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, archiveStatus); } /** @@ -397,16 +405,20 @@ private void importGroup(SQLiteDatabase database, CSVRecord record) throws Forma * Import a single card to group mapping into the database using the given * session. */ - private void importCardGroupMapping(SQLiteDatabase database, CSVRecord record) throws FormatException { + private void importCardGroupMapping(SQLiteDatabase database, CSVRecord record, int maxLoyaltyCardId) throws FormatException { int cardId = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIdsGroups.cardID, record); String groupId = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIdsGroups.groupID, record, null); + if (cardId < 1) { + throw new FormatException("Card ID must be >= 1"); + } if (groupId == null) { throw new FormatException("Group has no ID: " + record); } - List cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId); + int newCardId = cardId + maxLoyaltyCardId; + List cardGroups = DBHelper.getLoyaltyCardGroups(database, newCardId); cardGroups.add(DBHelper.getGroup(database, groupId)); - DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups); + DBHelper.setLoyaltyCardGroups(database, newCardId, cardGroups); } } diff --git a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java index df7fe18a8d..3b0a4942f3 100644 --- a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java @@ -75,6 +75,10 @@ public static ImportExportResult importData(Context context, SQLiteDatabase data Log.e(TAG, error); } + for (String fileName : newImageFiles) { + context.deleteFile(fileName); + } + return new ImportExportResult(ImportExportResultType.GenericFailure, error); } } diff --git a/app/src/test/java/protect/card_locker/ImportExportTest.java b/app/src/test/java/protect/card_locker/ImportExportTest.java index 3bd6adfe7d..b1e3752db2 100644 --- a/app/src/test/java/protect/card_locker/ImportExportTest.java +++ b/app/src/test/java/protect/card_locker/ImportExportTest.java @@ -201,7 +201,12 @@ private void addGroups(int groupsToAdd) { * where the smallest card's index is 1 */ private void checkLoyaltyCards() { + checkLoyaltyCards(false); + } + + private void checkLoyaltyCards(boolean dups) { Cursor cursor = DBHelper.getLoyaltyCardCursor(mDatabase); + boolean first = true; int index = 1; while (cursor.moveToNext()) { @@ -222,7 +227,16 @@ private void checkLoyaltyCards() { assertEquals(Integer.valueOf(index), card.headerColor); assertEquals(0, card.starStatus); - index++; + if (dups) { + if (first) { + first = false; + } else { + first = true; + index++; + } + } else { + index++; + } } cursor.close(); } @@ -500,9 +514,9 @@ public void importExistingCardsNotReplace() throws IOException { result = MultiFormatImporter.importData(activity.getApplicationContext(), mDatabase, inData, DataFormat.Catima, null); assertEquals(ImportExportResultType.Success, result.resultType()); - assertEquals(NUM_CARDS, DBHelper.getLoyaltyCardCount(mDatabase)); + assertEquals(NUM_CARDS * 2, DBHelper.getLoyaltyCardCount(mDatabase)); - checkLoyaltyCards(); + checkLoyaltyCards(true); // Clear the database for the next format under test TestHelpers.getEmptyDb(activity); @@ -764,7 +778,7 @@ public void importWithInvalidStarFieldV1() { // Import the CSV data result = MultiFormatImporter.importData(activity.getApplicationContext(), mDatabase, inputStream, DataFormat.Catima, null); assertEquals(ImportExportResultType.Success, result.resultType()); - assertEquals(1, DBHelper.getLoyaltyCardCount(mDatabase)); + assertEquals(2, DBHelper.getLoyaltyCardCount(mDatabase)); LoyaltyCard card = DBHelper.getLoyaltyCard(mDatabase, 1);