[libeufin] 01/02: Disk-space policy: fix database query.

[gnunet]

This is an automated email from the git hooks/post-receive script.

ms pushed a commit to branch master
in repository libeufin.

commit 812122ba1de2b7eb4bb52c11df05db3c9bf01675
Author: MS <ms@taler.net>
AuthorDate: Tue Dec 13 22:58:32 2022 +0100

    Disk-space policy: fix database query.
    
    Revert use of index-based search, because the
    deletion of bank messages is now optional.  That
    avoids to process stored messages more than once.
---
 nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt    |  4 +++
 .../tech/libeufin/nexus/bankaccount/BankAccount.kt | 42 ++++++++++++++--------
 .../kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt |  1 +
 nexus/src/test/kotlin/MakeEnv.kt                   |  2 ++
 4 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
index 4c6dc18e..a120da7b 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -285,6 +285,8 @@ object NexusBankAccountsTable : LongIdTable() {
     val lastStatementCreationTimestamp = 
long("lastStatementCreationTimestamp").nullable()
     val lastReportCreationTimestamp = 
long("lastReportCreationTimestamp").nullable()
     val lastNotificationCreationTimestamp = 
long("lastNotificationCreationTimestamp").nullable()
+    // Highest bank message ID that this bank account is aware of.
+    val highestSeenBankMessageSerialId = long("highestSeenBankMessageSerialId")
     val pain001Counter = long("pain001counter").default(1)
 }
 
@@ -294,11 +296,13 @@ class NexusBankAccountEntity(id: EntityID<Long>) : 
LongEntity(id) {
             return find { NexusBankAccountsTable.bankAccountName eq name 
}.firstOrNull()
         }
     }
+
     var bankAccountName by NexusBankAccountsTable.bankAccountName
     var accountHolder by NexusBankAccountsTable.accountHolder
     var iban by NexusBankAccountsTable.iban
     var bankCode by NexusBankAccountsTable.bankCode
     var defaultBankConnection by NexusBankConnectionEntity 
optionalReferencedOn NexusBankAccountsTable.defaultBankConnection
+    var highestSeenBankMessageSerialId by 
NexusBankAccountsTable.highestSeenBankMessageSerialId
     var pain001Counter by NexusBankAccountsTable.pain001Counter
     var lastStatementCreationTimestamp by 
NexusBankAccountsTable.lastStatementCreationTimestamp
     var lastReportCreationTimestamp by 
NexusBankAccountsTable.lastReportCreationTimestamp
diff --git 
a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
index 6025e5ee..87150a24 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -39,14 +39,14 @@ import java.time.Instant
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
-// Defaults to false.  Gets true if defined as "yes".
 private val keepBankMessages: String? = 
System.getenv("LIBEUFIN_NEXUS_KEEP_BANK_MESSAGES")
-
 fun requireBankAccount(call: ApplicationCall, parameterKey: String): 
NexusBankAccountEntity {
     val name = call.parameters[parameterKey]
-    if (name == null) {
-        throw NexusError(HttpStatusCode.InternalServerError, "no parameter for 
bank account")
-    }
+    if (name == null)
+        throw NexusError(
+            HttpStatusCode.InternalServerError,
+            "no parameter for bank account"
+        )
     val account = transaction { NexusBankAccountEntity.findByName(name) }
     if (account == null) {
         throw NexusError(HttpStatusCode.NotFound, "bank connection '$name' not 
found")
@@ -291,7 +291,10 @@ fun processCamtMessage(
  * Create new transactions for an account based on bank messages it
  * did not see before.
  */
-fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: 
String): CamtTransactionsCount {
+fun ingestBankMessagesIntoAccount(
+    bankConnectionId: String,
+    bankAccountId: String
+): CamtTransactionsCount {
     var totalNew = 0
     var downloadedTransactions = 0
     transaction {
@@ -304,25 +307,35 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: 
String, bankAccountId: Strin
         if (acct == null) {
             throw NexusError(HttpStatusCode.InternalServerError, "account not 
found")
         }
+        var lastId = acct.highestSeenBankMessageSerialId
         NexusBankMessageEntity.find {
-            (NexusBankMessagesTable.bankConnection eq conn.id) and not(
-                NexusBankMessagesTable.errors
-            )
+            (NexusBankMessagesTable.bankConnection eq conn.id) and
+                    (NexusBankMessagesTable.id greater 
acct.highestSeenBankMessageSerialId) and
+                    // Wrong messages got already skipped by the
+                    // index check above.  Below is a extra check.
+                    not(NexusBankMessagesTable.errors)
         }.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
             val doc = 
XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
             val processingResult = processCamtMessage(bankAccountId, doc, 
it.code)
             if (processingResult.newTransactions == -1) {
                 it.errors = true
+                lastId = it.id.value
                 return@forEach
             }
-            /**
-             * The XML document from the bank parsed correctly,
-             * remove now from the database.
-             */
-            if (keepBankMessages == null || keepBankMessages != "yes") 
it.delete()
             totalNew += processingResult.newTransactions
             downloadedTransactions += processingResult.downloadedTransactions
+            /**
+             * Disk-space conservative check: only store if "yes" was
+             * explicitly set into the environment variable.  Any other
+             * value or non given falls back to deletion.
+             */
+            if (keepBankMessages == null || keepBankMessages != "yes") {
+                it.delete()
+                return@forEach
+            }
+            lastId = it.id.value
         }
+        acct.highestSeenBankMessageSerialId = lastId
     }
     // return totalNew
     return CamtTransactionsCount(
@@ -449,6 +462,7 @@ fun importBankAccount(call: ApplicationCall, 
offeredBankAccountId: String, nexus
                         iban = offeredAccount[OfferedBankAccountsTable.iban]
                         bankCode = 
offeredAccount[OfferedBankAccountsTable.bankCode]
                         defaultBankConnection = conn
+                        highestSeenBankMessageSerialId = 0
                         accountHolder = 
offeredAccount[OfferedBankAccountsTable.accountHolder]
                     }
                     logger.info("Account ${newImportedAccount.id} gets 
imported")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index 0cc58106..a8069977 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -305,6 +305,7 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
                                     reason = "bank gave no BIC"
                                 )
                             defaultBankConnection = conn
+                            highestSeenBankMessageSerialId = 0
                         }
                     }
                 }
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
index 1eab2016..58b2affb 100644
--- a/nexus/src/test/kotlin/MakeEnv.kt
+++ b/nexus/src/test/kotlin/MakeEnv.kt
@@ -89,6 +89,7 @@ fun prepNexusDb() {
             iban = FOO_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
+            highestSeenBankMessageSerialId = 0
             accountHolder = "foo"
         }
         val b = NexusBankAccountEntity.new {
@@ -96,6 +97,7 @@ fun prepNexusDb() {
             iban = BAR_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
+            highestSeenBankMessageSerialId = 0
             accountHolder = "bar"
         }
     }

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.