[libeufin] 02/02: Disk space policy at Nexus.

[gnunet]

This is an automated email from the git hooks/post-receive script.

ms pushed a commit to branch master
in repository libeufin.

commit cdf522a9c77db2547cab2fc26919f48a1bb606e7
Author: MS <ms@taler.net>
AuthorDate: Tue Dec 13 20:12:53 2022 +0100

    Disk space policy at Nexus.
    
    Make the conservative storing of bank messages
    optional, defaulting to non storing them.
---
 nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt         |  5 -----
 .../tech/libeufin/nexus/bankaccount/BankAccount.kt      | 17 +++++++++++------
 .../main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt |  1 -
 nexus/src/test/kotlin/MakeEnv.kt                        |  2 --
 4 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
index 30192362..4c6dc18e 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -285,9 +285,6 @@ object NexusBankAccountsTable : LongIdTable() {
     val lastStatementCreationTimestamp = 
long("lastStatementCreationTimestamp").nullable()
     val lastReportCreationTimestamp = 
long("lastReportCreationTimestamp").nullable()
     val lastNotificationCreationTimestamp = 
long("lastNotificationCreationTimestamp").nullable()
-
-    // Highest bank message ID that this bank account is aware of.
-    val highestSeenBankMessageSerialId = long("highestSeenBankMessageSerialId")
     val pain001Counter = long("pain001counter").default(1)
 }
 
@@ -297,13 +294,11 @@ class NexusBankAccountEntity(id: EntityID<Long>) : 
LongEntity(id) {
             return find { NexusBankAccountsTable.bankAccountName eq name 
}.firstOrNull()
         }
     }
-
     var bankAccountName by NexusBankAccountsTable.bankAccountName
     var accountHolder by NexusBankAccountsTable.accountHolder
     var iban by NexusBankAccountsTable.iban
     var bankCode by NexusBankAccountsTable.bankCode
     var defaultBankConnection by NexusBankConnectionEntity 
optionalReferencedOn NexusBankAccountsTable.defaultBankConnection
-    var highestSeenBankMessageSerialId by 
NexusBankAccountsTable.highestSeenBankMessageSerialId
     var pain001Counter by NexusBankAccountsTable.pain001Counter
     var lastStatementCreationTimestamp by 
NexusBankAccountsTable.lastStatementCreationTimestamp
     var lastReportCreationTimestamp by 
NexusBankAccountsTable.lastReportCreationTimestamp
diff --git 
a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
index 102feaa2..6025e5ee 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -39,6 +39,9 @@ import java.time.Instant
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
+// Defaults to false.  Gets true if defined as "yes".
+private val keepBankMessages: String? = 
System.getenv("LIBEUFIN_NEXUS_KEEP_BANK_MESSAGES")
+
 fun requireBankAccount(call: ApplicationCall, parameterKey: String): 
NexusBankAccountEntity {
     val name = call.parameters[parameterKey]
     if (name == null) {
@@ -301,10 +304,10 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: 
String, bankAccountId: Strin
         if (acct == null) {
             throw NexusError(HttpStatusCode.InternalServerError, "account not 
found")
         }
-        var lastId = acct.highestSeenBankMessageSerialId
         NexusBankMessageEntity.find {
-            (NexusBankMessagesTable.bankConnection eq conn.id) and
-                    (NexusBankMessagesTable.id greater 
acct.highestSeenBankMessageSerialId)
+            (NexusBankMessagesTable.bankConnection eq conn.id) and not(
+                NexusBankMessagesTable.errors
+            )
         }.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
             val doc = 
XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
             val processingResult = processCamtMessage(bankAccountId, doc, 
it.code)
@@ -312,11 +315,14 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: 
String, bankAccountId: Strin
                 it.errors = true
                 return@forEach
             }
-            lastId = it.id.value
+            /**
+             * The XML document from the bank parsed correctly,
+             * remove now from the database.
+             */
+            if (keepBankMessages == null || keepBankMessages != "yes") 
it.delete()
             totalNew += processingResult.newTransactions
             downloadedTransactions += processingResult.downloadedTransactions
         }
-        acct.highestSeenBankMessageSerialId = lastId
     }
     // return totalNew
     return CamtTransactionsCount(
@@ -443,7 +449,6 @@ fun importBankAccount(call: ApplicationCall, 
offeredBankAccountId: String, nexus
                         iban = offeredAccount[OfferedBankAccountsTable.iban]
                         bankCode = 
offeredAccount[OfferedBankAccountsTable.bankCode]
                         defaultBankConnection = conn
-                        highestSeenBankMessageSerialId = 0
                         accountHolder = 
offeredAccount[OfferedBankAccountsTable.accountHolder]
                     }
                     logger.info("Account ${newImportedAccount.id} gets 
imported")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index a8069977..0cc58106 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -305,7 +305,6 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
                                     reason = "bank gave no BIC"
                                 )
                             defaultBankConnection = conn
-                            highestSeenBankMessageSerialId = 0
                         }
                     }
                 }
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
index 58b2affb..1eab2016 100644
--- a/nexus/src/test/kotlin/MakeEnv.kt
+++ b/nexus/src/test/kotlin/MakeEnv.kt
@@ -89,7 +89,6 @@ fun prepNexusDb() {
             iban = FOO_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
-            highestSeenBankMessageSerialId = 0
             accountHolder = "foo"
         }
         val b = NexusBankAccountEntity.new {
@@ -97,7 +96,6 @@ fun prepNexusDb() {
             iban = BAR_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
-            highestSeenBankMessageSerialId = 0
             accountHolder = "bar"
         }
     }

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.