[libeufin] branch master updated (cdf522a9 -> 7f81b491)

[gnunet]

This is an automated email from the git hooks/post-receive script.

ms pushed a change to branch master
in repository libeufin.

    from cdf522a9 Disk space policy at Nexus.
     new 812122ba Disk-space policy: fix database query.
     new 7f81b491 Disk-space policy at Sandbox.

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt    |  4 ++
 .../tech/libeufin/nexus/bankaccount/BankAccount.kt | 42 ++++++++++++-------
 .../kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt |  1 +
 nexus/src/test/kotlin/MakeEnv.kt                   |  2 +
 .../tech/libeufin/sandbox/EbicsProtocolBackend.kt  | 49 ++++++++++++++++------
 .../src/main/kotlin/tech/libeufin/sandbox/Main.kt  |  4 +-
 6 files changed, 73 insertions(+), 29 deletions(-)

diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
index 4c6dc18e..a120da7b 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -285,6 +285,8 @@ object NexusBankAccountsTable : LongIdTable() {
     val lastStatementCreationTimestamp = 
long("lastStatementCreationTimestamp").nullable()
     val lastReportCreationTimestamp = 
long("lastReportCreationTimestamp").nullable()
     val lastNotificationCreationTimestamp = 
long("lastNotificationCreationTimestamp").nullable()
+    // Highest bank message ID that this bank account is aware of.
+    val highestSeenBankMessageSerialId = long("highestSeenBankMessageSerialId")
     val pain001Counter = long("pain001counter").default(1)
 }
 
@@ -294,11 +296,13 @@ class NexusBankAccountEntity(id: EntityID<Long>) : 
LongEntity(id) {
             return find { NexusBankAccountsTable.bankAccountName eq name 
}.firstOrNull()
         }
     }
+
     var bankAccountName by NexusBankAccountsTable.bankAccountName
     var accountHolder by NexusBankAccountsTable.accountHolder
     var iban by NexusBankAccountsTable.iban
     var bankCode by NexusBankAccountsTable.bankCode
     var defaultBankConnection by NexusBankConnectionEntity 
optionalReferencedOn NexusBankAccountsTable.defaultBankConnection
+    var highestSeenBankMessageSerialId by 
NexusBankAccountsTable.highestSeenBankMessageSerialId
     var pain001Counter by NexusBankAccountsTable.pain001Counter
     var lastStatementCreationTimestamp by 
NexusBankAccountsTable.lastStatementCreationTimestamp
     var lastReportCreationTimestamp by 
NexusBankAccountsTable.lastReportCreationTimestamp
diff --git 
a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
index 6025e5ee..87150a24 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -39,14 +39,14 @@ import java.time.Instant
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
-// Defaults to false.  Gets true if defined as "yes".
 private val keepBankMessages: String? = 
System.getenv("LIBEUFIN_NEXUS_KEEP_BANK_MESSAGES")
-
 fun requireBankAccount(call: ApplicationCall, parameterKey: String): 
NexusBankAccountEntity {
     val name = call.parameters[parameterKey]
-    if (name == null) {
-        throw NexusError(HttpStatusCode.InternalServerError, "no parameter for 
bank account")
-    }
+    if (name == null)
+        throw NexusError(
+            HttpStatusCode.InternalServerError,
+            "no parameter for bank account"
+        )
     val account = transaction { NexusBankAccountEntity.findByName(name) }
     if (account == null) {
         throw NexusError(HttpStatusCode.NotFound, "bank connection '$name' not 
found")
@@ -291,7 +291,10 @@ fun processCamtMessage(
  * Create new transactions for an account based on bank messages it
  * did not see before.
  */
-fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: 
String): CamtTransactionsCount {
+fun ingestBankMessagesIntoAccount(
+    bankConnectionId: String,
+    bankAccountId: String
+): CamtTransactionsCount {
     var totalNew = 0
     var downloadedTransactions = 0
     transaction {
@@ -304,25 +307,35 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: 
String, bankAccountId: Strin
         if (acct == null) {
             throw NexusError(HttpStatusCode.InternalServerError, "account not 
found")
         }
+        var lastId = acct.highestSeenBankMessageSerialId
         NexusBankMessageEntity.find {
-            (NexusBankMessagesTable.bankConnection eq conn.id) and not(
-                NexusBankMessagesTable.errors
-            )
+            (NexusBankMessagesTable.bankConnection eq conn.id) and
+                    (NexusBankMessagesTable.id greater 
acct.highestSeenBankMessageSerialId) and
+                    // Wrong messages got already skipped by the
+                    // index check above.  Below is a extra check.
+                    not(NexusBankMessagesTable.errors)
         }.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
             val doc = 
XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
             val processingResult = processCamtMessage(bankAccountId, doc, 
it.code)
             if (processingResult.newTransactions == -1) {
                 it.errors = true
+                lastId = it.id.value
                 return@forEach
             }
-            /**
-             * The XML document from the bank parsed correctly,
-             * remove now from the database.
-             */
-            if (keepBankMessages == null || keepBankMessages != "yes") 
it.delete()
             totalNew += processingResult.newTransactions
             downloadedTransactions += processingResult.downloadedTransactions
+            /**
+             * Disk-space conservative check: only store if "yes" was
+             * explicitly set into the environment variable.  Any other
+             * value or non given falls back to deletion.
+             */
+            if (keepBankMessages == null || keepBankMessages != "yes") {
+                it.delete()
+                return@forEach
+            }
+            lastId = it.id.value
         }
+        acct.highestSeenBankMessageSerialId = lastId
     }
     // return totalNew
     return CamtTransactionsCount(
@@ -449,6 +462,7 @@ fun importBankAccount(call: ApplicationCall, 
offeredBankAccountId: String, nexus
                         iban = offeredAccount[OfferedBankAccountsTable.iban]
                         bankCode = 
offeredAccount[OfferedBankAccountsTable.bankCode]
                         defaultBankConnection = conn
+                        highestSeenBankMessageSerialId = 0
                         accountHolder = 
offeredAccount[OfferedBankAccountsTable.accountHolder]
                     }
                     logger.info("Account ${newImportedAccount.id} gets 
imported")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index 0cc58106..a8069977 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -305,6 +305,7 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
                                     reason = "bank gave no BIC"
                                 )
                             defaultBankConnection = conn
+                            highestSeenBankMessageSerialId = 0
                         }
                     }
                 }
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
index 1eab2016..58b2affb 100644
--- a/nexus/src/test/kotlin/MakeEnv.kt
+++ b/nexus/src/test/kotlin/MakeEnv.kt
@@ -89,6 +89,7 @@ fun prepNexusDb() {
             iban = FOO_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
+            highestSeenBankMessageSerialId = 0
             accountHolder = "foo"
         }
         val b = NexusBankAccountEntity.new {
@@ -96,6 +97,7 @@ fun prepNexusDb() {
             iban = BAR_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
+            highestSeenBankMessageSerialId = 0
             accountHolder = "bar"
         }
     }
diff --git 
a/sandbox/src/main/kotlin/tech/libeufin/sandbox/EbicsProtocolBackend.kt 
b/sandbox/src/main/kotlin/tech/libeufin/sandbox/EbicsProtocolBackend.kt
index a50d99b2..bb95b837 100644
--- a/sandbox/src/main/kotlin/tech/libeufin/sandbox/EbicsProtocolBackend.kt
+++ b/sandbox/src/main/kotlin/tech/libeufin/sandbox/EbicsProtocolBackend.kt
@@ -508,7 +508,8 @@ fun getLastBalance(bankAccount: BankAccountEntity): 
BigDecimal {
         BankAccountStatementsTable.bankAccount eq bankAccount.id
     }.firstOrNull()
     val lastBalance = if (lastStatement == null) {
-        BigDecimal.ZERO } else { BigDecimal(lastStatement.balanceClbd) }
+        BigDecimal.ZERO
+    } else { BigDecimal(lastStatement.balanceClbd) }
     return lastBalance
 }
 
@@ -520,17 +521,28 @@ fun getLastBalance(bankAccount: BankAccountEntity): 
BigDecimal {
 private fun constructCamtResponse(
     type: Int,
     subscriber: EbicsSubscriberEntity,
-    dateRange: Pair<Long, Long>?): List<String> {
+    dateRange: Pair<Long, Long>?
+): List<String> {
 
     if (type != 53 && type != 52) throw EbicsUnsupportedOrderType()
     val bankAccount = getBankAccountFromSubscriber(subscriber)
     if (type == 52) {
+        if (dateRange != null)
+            throw NotImplementedError() // FIXME: #6243.
+        /**
+         * Note: before addressing #6243, the C52 is always generated
+         * without taking the time range into consideration.  That means
+         * that the request is treated always as "give last non booked"
+         * transactions.  The current implementation returns non booked
+         * transactions only on the first request, when the time range is
+         * missing.
+         */
         val history = mutableListOf<RawPayment>()
         /**
-         * This block adds all the fresh transactions to the intermediate
+         * This block adds all the non booked transactions to the intermediate
          * history list and returns the last balance that was reported in a
          * C53 document.  This latter will be the base balance to calculate
-         * the final balance after the fresh transactions.
+         * the final balance after the non booked transactions.
          */
         val lastBalance = transaction {
             BankAccountFreshTransactionEntity.all().forEach {
@@ -559,13 +571,9 @@ private fun constructCamtResponse(
             ret
         } else ""
         logger.debug("camt.052 document '${camtData.messageId}' 
generated.$payments")
-        return listOf(
-            camtData.camtMessage
-        )
+        return listOf(camtData.camtMessage)
     }
     SandboxAssert(type == 53, "Didn't catch unsupported CAMT type")
-    logger.debug("Finding C$type records")
-
     /**
      * FIXME: when this function throws an exception, it makes a JSON response 
being responded.
      * That is bad, because here we're inside a Ebics handler and only XML 
should
@@ -575,10 +583,10 @@ private fun constructCamtResponse(
     val ret = mutableListOf<String>()
     /**
      * Retrieve all the records whose creation date lies into the
-     * time range given as a function's parameter.
+     * time range given in the function parameters.
      */
     if (dateRange != null) {
-        logger.debug("Querying C$type with date range: $dateRange")
+        logger.debug("Querying C53 with date range: $dateRange")
         BankAccountStatementEntity.find {
             BankAccountStatementsTable.creationTime.between(
                 dateRange.first,
@@ -589,7 +597,6 @@ private fun constructCamtResponse(
         /**
          * No time range was given, hence pick the latest statement.
          */
-        logger.debug("No date range was given for c$type, respond with latest 
document")
         BankAccountStatementEntity.find {
             BankAccountStatementsTable.bankAccount eq bankAccount.id
         }.lastOrNull().apply {
@@ -1445,10 +1452,26 @@ suspend fun ApplicationCall.ebicsweb() {
                             throw EbicsInvalidRequestError()
                         logger.debug("Handling download receipt for EBICS 
transaction: " +
                                 requestTransactionID)
+                        /**
+                         * The receipt phase means that the client has already
+                         * received all the data related to the current 
download
+                         * transaction.  Hence this data can now be removed 
from
+                         * the database.
+                         */
+                        val ebicsData = transaction {
+                            
EbicsDownloadTransactionEntity.findById(requestTransactionID)
+                        }
+                        if (ebicsData == null)
+                            throw SandboxError(
+                                HttpStatusCode.InternalServerError,
+                                "EBICS transaction $requestTransactionID was 
not" +
+                                        "found in the database for deletion.",
+                                
LibeufinErrorCode.LIBEUFIN_EC_INCONSISTENT_STATE
+                            )
+                        ebicsData.delete()
                         val receiptCode =
                             requestObject.body.transferReceipt?.receiptCode ?: 
throw EbicsInvalidRequestError()
                         
EbicsResponse.createForDownloadReceiptPhase(requestTransactionID, receiptCode 
== 0)
-
                     }
                 }
                 signEbicsResponse(ebicsResponse, requestContext.hostAuthPriv)
diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt 
b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt
index 52d0d86a..c4742f2d 100644
--- a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt
+++ b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt
@@ -1005,8 +1005,8 @@ val sandboxApp: Application.() -> Unit = {
                 throw EbicsProcessingError("Serving EBICS threw unmanaged 
UtilError: ${e.reason}")
             }
             catch (e: SandboxError) {
-                val payload: String = e.message ?: e.stackTraceToString()
-                logger.info(payload)
+                val errorInfo: String = e.message ?: e.stackTraceToString()
+                logger.info(errorInfo)
                 // Should translate to EBICS error code.
                 when (e.errorCode) {
                     LibeufinErrorCode.LIBEUFIN_EC_INVALID_STATE -> throw 
EbicsProcessingError("Invalid bank state.")

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.