Add BI source integration and update rate source configs

README.md

@@ -2,7 +2,7 @@
 
 Rate-boss periodically pulls exchange rates from external sources, stores them in the database, and optionally publishes rate events to Kafka. It also exposes API endpoints to read the latest or historical rates and to convert amounts by timestamp.
 
-Supported sources today: Fixer, CBR, NBKZ.
+Supported sources today: Fixer, CBR, BI, NBKZ.
 
 ## Service flow (actual)
 
@@ -27,3 +27,8 @@ For each configured source:
 - `ExchangeGrabberMasterJob` reads the list of base currencies from `rates.*Job.currencies`.
 - It spawns `ExchangeGrabberJob` per currency.
 - Each `ExchangeGrabberJob` calls the respective `ExchangeRateSource`, saves the rates, and (for some sources) emits Kafka events.
+
+## BI source notes
+
+- Source: Bank Indonesia
+- Stored BI rate format: `target currency per 1000 IDR` (e.g. `USD per 1000 IDR`), i.e. `1000 / (jual_subkurslokal / nil_subkurslokal)`.

src/main/kotlin/dev/vality/rateboss/client/bi/BiApiClient.kt

@@ -0,0 +1,49 @@
+package dev.vality.rateboss.client.bi
+
+import dev.vality.rateboss.config.properties.RatesProperties
+import org.springframework.http.HttpEntity
+import org.springframework.http.HttpHeaders
+import org.springframework.http.HttpMethod
+import org.springframework.stereotype.Component
+import org.springframework.web.client.RestTemplate
+import org.springframework.web.client.exchange
+import org.springframework.web.util.UriComponentsBuilder
+import java.time.LocalDate
+import java.time.format.DateTimeFormatter
+
+@Component
+class BiApiClient(
+    private val restTemplate: RestTemplate,
+    private val ratesProperties: RatesProperties,
+) {
+    fun getExchangeRates(
+        currencySymbolCode: String,
+        startDate: LocalDate,
+        endDate: LocalDate,
+    ): String {
+        val url = buildUrl(currencySymbolCode, startDate, endDate)
+        val requestHeaders = HttpHeaders()
+        requestHeaders["User-Agent"] = BROWSER_USER_AGENT
+        return restTemplate.exchange<String>(url, HttpMethod.GET, HttpEntity<Any>(requestHeaders)).body!!
+    }
+
+    private fun buildUrl(
+        currencySymbolCode: String,
+        startDate: LocalDate,
+        endDate: LocalDate,
+    ): String =
+        UriComponentsBuilder
+            .fromUriString(ratesProperties.source.bi.rootUrl)
+            .queryParam("mts", currencySymbolCode)
+            .queryParam("startdate", startDate.format(BI_DATE_FORMATTER))
+            .queryParam("enddate", endDate.format(BI_DATE_FORMATTER))
+            .build()
+            .toUriString()
+
+    companion object {
+        private val BI_DATE_FORMATTER: DateTimeFormatter = DateTimeFormatter.ofPattern("dd-MM-yyyy")
+        private const val BROWSER_USER_AGENT =
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+    }
+}

src/main/kotlin/dev/vality/rateboss/config/JobConfig.kt

@@ -1,6 +1,7 @@
 package dev.vality.rateboss.config
 
 import dev.vality.rateboss.config.properties.RatesProperties
+import dev.vality.rateboss.job.BiExchangeGrabberMasterJob
 import dev.vality.rateboss.job.CbrExchangeGrabberMasterJob
 import dev.vality.rateboss.job.FixerExchangeGrabberMasterJob
 import dev.vality.rateboss.job.NbazExchangeGrabberMasterJob
@@ -47,6 +48,16 @@ class JobConfig {
                 schedulerFactoryBean.triggerJob(JobKey(ratesProperties.cbrJob.jobKey))
             }
         }
+        val biJobTriggerName = ratesProperties.biJob.jobTriggerName
+        if (biJobTriggerName.isNotEmpty()) {
+            schedulerFactoryBean.addJob(biExchangeRateGrabberMasterJob(), true, true)
+            if (!schedulerFactoryBean.checkExists(TriggerKey(ratesProperties.biJob.jobTriggerName))) {
+                schedulerFactoryBean.scheduleJob(biExchangeRateGrabberMasterTrigger())
+            }
+            if (runOnStartup) {
+                schedulerFactoryBean.triggerJob(JobKey(ratesProperties.biJob.jobKey))
+            }
+        }
         val nbkzJobTriggerName = ratesProperties.nbkzJob.jobTriggerName
         if (nbkzJobTriggerName.isNotEmpty()) {
             schedulerFactoryBean.addJob(nbkzExchangeRateGrabberMasterJob(), true, true)
@@ -119,6 +130,21 @@ class JobConfig {
             .withSchedule(CronScheduleBuilder.cronSchedule(ratesProperties.cbrJob.jobCron))
             .build()
 
+    fun biExchangeRateGrabberMasterJob(): JobDetailImpl {
+        val jobDetail = JobDetailImpl()
+        jobDetail.key = JobKey(ratesProperties.biJob.jobKey)
+        jobDetail.jobClass = BiExchangeGrabberMasterJob::class.java
+        return jobDetail
+    }
+
+    fun biExchangeRateGrabberMasterTrigger(): CronTrigger =
+        TriggerBuilder
+            .newTrigger()
+            .forJob(biExchangeRateGrabberMasterJob())
+            .withIdentity(ratesProperties.biJob.jobTriggerName)
+            .withSchedule(CronScheduleBuilder.cronSchedule(ratesProperties.biJob.jobCron))
+            .build()
+
     fun nbkzExchangeRateGrabberMasterJob(): JobDetailImpl {
         val jobDetail = JobDetailImpl()
         jobDetail.key = JobKey(ratesProperties.nbkzJob.jobKey)

src/main/kotlin/dev/vality/rateboss/config/properties/RatesProperties.kt

@@ -9,6 +9,7 @@ import java.time.ZoneId
 data class RatesProperties(
     val fixerJob: JobDescription,
     val cbrJob: JobDescription,
+    val biJob: JobDescription,
     val nbkzJob: JobDescription,
     val nbkrJob: JobDescription,
     val nbuzJob: JobDescription,
@@ -31,6 +32,7 @@ data class CurrencyProperties(
 data class RatesSourceProperties(
     val fixer: FixerProperties,
     val cbr: CbrProperties,
+    val bi: BiProperties,
     val nbkz: NbkzProperties,
     val nbkr: NbkrProperties,
     val nbuz: NbuzProperties,
@@ -47,6 +49,13 @@ data class CbrProperties(
     val timeZone: ZoneId,
 )
 
+data class BiProperties(
+    val rootUrl: String,
+    val timeZone: ZoneId,
+    val lookbackDays: Long,
+    val targetCurrencies: List<String>,
+)
+
 data class NbkzProperties(
     val rootUrl: String,
     val dateFormat: String,

src/main/kotlin/dev/vality/rateboss/job/BiExchangeGrabberJob.kt

@@ -0,0 +1,37 @@
+package dev.vality.rateboss.job
+
+import dev.vality.rateboss.extensions.getApplicationContext
+import dev.vality.rateboss.source.ExchangeRateSource
+import dev.vality.rateboss.source.ExchangeRateSourceException
+import dev.vality.rateboss.source.impl.BiExchangeRateSource
+import dev.vality.rateboss.source.model.ExchangeRates
+import mu.KotlinLogging
+import org.quartz.JobExecutionContext
+import org.springframework.context.ApplicationContext
+import org.springframework.retry.support.RetryTemplate
+
+private val log = KotlinLogging.logger {}
+
+class BiExchangeGrabberJob : AbstractExchangeGrabberJob() {
+    override fun executeInternal(context: JobExecutionContext) {
+        val applicationContext = context.getApplicationContext()
+        val currencySymbolCode = context.jobDetail.jobDataMap["currencySymbolCode"] as String
+        val currencyExponent = context.jobDetail.jobDataMap["currencyExponent"] as Int
+        val exchangeRateSource = applicationContext.getBean(BiExchangeRateSource::class.java)
+        val sourceId = exchangeRateSource.getSourceId()
+        log.info { "Process BiExchangeGrabberJob for $sourceId" }
+        val exchangeRates = getExchangeRates(applicationContext, exchangeRateSource, currencySymbolCode)
+        saveExchangeRates(applicationContext, currencySymbolCode, currencyExponent, exchangeRates, sourceId)
+    }
+
+    private fun getExchangeRates(
+        applicationContext: ApplicationContext,
+        exchangeRateSource: ExchangeRateSource,
+        currencySymbolCode: String,
+    ): ExchangeRates {
+        val retryTemplate = applicationContext.getBean(RetryTemplate::class.java)
+        return retryTemplate.execute<ExchangeRates, ExchangeRateSourceException> {
+            exchangeRateSource.getExchangeRate(currencySymbolCode)
+        }
+    }
+}

src/main/kotlin/dev/vality/rateboss/job/BiExchangeGrabberMasterJob.kt

@@ -0,0 +1,18 @@
+package dev.vality.rateboss.job
+
+import dev.vality.rateboss.config.properties.RatesProperties
+import dev.vality.rateboss.extensions.getApplicationContext
+import org.quartz.JobExecutionContext
+import org.quartz.Scheduler
+
+class BiExchangeGrabberMasterJob : AbstractExchangeGrabberMasterJob() {
+    override fun executeInternal(context: JobExecutionContext) {
+        val applicationContext = context.getApplicationContext()
+        val ratesProperties = applicationContext.getBean(RatesProperties::class.java)
+        val currencies = ratesProperties.biJob.currencies
+        val schedulerFactoryBean = applicationContext.getBean(Scheduler::class.java)
+        launchJob(currencies, schedulerFactoryBean, BiExchangeGrabberJob::class.java, getJobName())
+    }
+
+    override fun getJobName(): String = "biJob"
+}

src/main/kotlin/dev/vality/rateboss/job/constant/ExRateSources.kt

@@ -3,6 +3,7 @@ package dev.vality.rateboss.job.constant
 object ExRateSources {
     const val FIXER = "fixer"
     const val CBR = "cbr"
+    const val BI = "bi"
     const val NBKZ = "nbkz"
     const val NBKR = "nbkr"
     const val NBUZ = "nbuz"

src/main/kotlin/dev/vality/rateboss/source/impl/BiExchangeRateSource.kt

@@ -0,0 +1,141 @@
+package dev.vality.rateboss.source.impl
+
+import dev.vality.rateboss.client.bi.BiApiClient
+import dev.vality.rateboss.config.properties.RatesProperties
+import dev.vality.rateboss.job.constant.ExRateSources
+import dev.vality.rateboss.source.ExchangeRateSource
+import dev.vality.rateboss.source.ExchangeRateSourceException
+import dev.vality.rateboss.source.model.ExchangeRates
+import mu.KotlinLogging
+import org.springframework.stereotype.Component
+import org.w3c.dom.Element
+import org.w3c.dom.Node
+import java.io.ByteArrayInputStream
+import java.math.BigDecimal
+import java.math.MathContext
+import java.nio.charset.StandardCharsets
+import java.time.LocalDate
+import java.time.OffsetDateTime
+import java.time.ZoneOffset
+import javax.xml.parsers.DocumentBuilderFactory
+
+private val log = KotlinLogging.logger {}
+
+@Component
+class BiExchangeRateSource(
+    private val biApiClient: BiApiClient,
+    private val ratesProperties: RatesProperties,
+) : ExchangeRateSource {
+    override fun getExchangeRate(currencySymbolCode: String): ExchangeRates {
+        val timeZone = ratesProperties.source.bi.timeZone
+        val date = LocalDate.now(timeZone)
+        val lookbackDays = ratesProperties.source.bi.lookbackDays
+        val targetCurrencies = ratesProperties.source.bi.targetCurrencies
+        val startDate = date.minusDays(lookbackDays)
+        log.info { "Trying to get exchange rates from bi for currency=$currencySymbolCode, date=$date" }
+
+        val rates = mutableMapOf<String, BigDecimal>()
+        for (targetCurrency in targetCurrencies) {
+            val response =
+                try {
+                    biApiClient.getExchangeRates(
+                        currencySymbolCode = targetCurrency,
+                        startDate = startDate,
+                        endDate = date,
+                    )
+                } catch (e: Exception) {
+                    throw ExchangeRateSourceException("Remote client exception", e)
+                }
+
+            val parsedRate =
+                try {
+                    parseRate(response, targetCurrency)
+                } catch (e: Exception) {
+                    throw ExchangeRateSourceException("Failed to parse response from BiApi", e)
+                }
+            if (parsedRate != null) {
+                rates[targetCurrency] = parsedRate
+            }
+        }
+
+        if (rates.isEmpty()) {
+            throw ExchangeRateSourceException("Unsuccessful response from BiApi for period $startDate..$date")
+        }
+
+        val nextDayTimestamp = date.plusDays(1).atStartOfDay().toEpochSecond(ZoneOffset.UTC)
+        log.info { "Exchange rates from bi have been retrieved, date=$date, exchangeRates=$rates, targetTimestamp=$nextDayTimestamp" }
+
+        return ExchangeRates(
+            rates = rates,
+            timestamp = nextDayTimestamp,
+        )
+    }
+
+    override fun getSourceId(): String = ExRateSources.BI
+
+    private fun parseRate(
+        xmlContent: String,
+        targetCurrency: String,
+    ): BigDecimal? {
+        val document =
+            DocumentBuilderFactory
+                .newInstance()
+                .newDocumentBuilder()
+                .parse(ByteArrayInputStream(xmlContent.toByteArray(StandardCharsets.UTF_8)))
+
+        val tables = document.getElementsByTagName("Table")
+        var lastTimestamp: OffsetDateTime? = null
+        var idrPerTargetCurrency: BigDecimal? = null
+
+        for (i in 0 until tables.length) {
+            val tableNode = tables.item(i)
+            if (tableNode.nodeType != Node.ELEMENT_NODE) {
+                continue
+            }
+            val table = tableNode as Element
+            val parsedDate =
+                extractTextByTag(table, "tgl_subkurslokal")
+                    ?.let { OffsetDateTime.parse(it.trim()) }
+                    ?: continue
+            val nominal = extractDecimalByTag(table, "nil_subkurslokal")
+            val sell = extractDecimalByTag(table, "jual_subkurslokal")
+            if (nominal == null || nominal <= BigDecimal.ZERO || sell == null || sell <= BigDecimal.ZERO) {
+                continue
+            }
+            val normalizedSell = sell.divide(nominal, MathContext.DECIMAL64)
+            if (lastTimestamp == null || parsedDate.isAfter(lastTimestamp)) {
+                lastTimestamp = parsedDate
+                idrPerTargetCurrency = normalizedSell
+            }
+        }
+
+        val rate = idrPerTargetCurrency ?: return null
+        log.debug { "BI rate parsed for targetCurrency=$targetCurrency: idrPerTargetCurrency=$rate" }
+        // Use 1000 here instead of 1.
+        return BI_RATE_MULTIPLIER.divide(rate, MathContext.DECIMAL64)
+    }
+
+    private fun extractDecimalByTag(
+        parent: Element,
+        tag: String,
+    ): BigDecimal? =
+        extractTextByTag(parent, tag)
+            ?.normalizeDecimal()
+            ?.toBigDecimalOrNull()
+
+    private fun extractTextByTag(
+        parent: Element,
+        tag: String,
+    ): String? = parent.getElementsByTagName(tag).item(0)?.textContent
+
+    private fun String?.normalizeDecimal(): String =
+        this
+            ?.replace(" ", "")
+            ?.replace(",", ".")
+            ?.trim()
+            .orEmpty()
+
+    companion object {
+        private val BI_RATE_MULTIPLIER: BigDecimal = BigDecimal.valueOf(1000L)
+    }
+}

src/main/resources/application.yml

@@ -90,6 +90,13 @@ rates:
     currencies:
       - symbolCode: "RUB"
         exponent: 2
+  biJob:
+    jobCron: '0 0 0/1 * * ?'
+    jobKey: 'bi-exchange-rate-grabber-master-job'
+    jobTriggerName: 'bi-exchange-rate-grabber-master-job-trigger'
+    currencies:
+      - symbolCode: "IDR"
+        exponent: 2
   nbkzJob:
     jobCron: '0 0 0/1 * * ?'
     jobKey: 'nbkz-exchange-rate-grabber-master-job'
@@ -125,6 +132,11 @@ rates:
     cbr:
       rootUrl: https://www.cbr.ru/scripts/XML_daily.asp
       timeZone: Europe/Moscow
+    bi:
+      rootUrl: https://www.bi.go.id/biwebservice/wskursbi.asmx/getSubKursLokal3
+      timeZone: Asia/Jakarta
+      lookbackDays: 7
+      targetCurrencies: [ "USD", "EUR" ]
     nbkz:
       rootUrl: https://nationalbank.kz/rss/get_rates.cfm
       dateFormat: dd.MM.yyyy