[RTI][SUB] Write the Ingestion Script for RTIs

.gitignore

@@ -0,0 +1,32 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Code coverage profiles and other test artifacts
+*.out
+coverage.*
+*.coverprofile
+profile.cov
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env
+
+# Editor/IDE
+# .idea/
+# .vscode/

data/RTI/Lanka Fabric Ltd/2026-03-18/1/request.csv

@@ -0,0 +1,2 @@
+Title,Description,Source,Sender,Receiver Institution,Receiver Position
+Title 1,Description 1,http://sourcelink,LDF,Lanka Fabric Ltd,Secretary

data/RTI/Lanka Fabric Ltd/2026-03-18/1/status.csv

@@ -0,0 +1,13 @@
+Status,Description,Entry Time,Exit Time,Source
+Request Identification,Request Identification Description,2022-09-14,2022-09-15,http://link
+Request Drafted,Request Drafted Description,2022-10-01,2022-10-02,http://link
+Draft Review,Draft Review Description,2022-10-03,2022-10-05,http://link
+Ready To Send,Ready To Send Description,2023-04-13,2023-04-21,http://link
+Request Sent,Request Sent Description,2023-04-14,2023-04-15,http://link
+Initial Request In Review,Initial Request In Review Description,2023-04-17,2023-04-18,http://link
+Initial Response,Initial Response Description,2023-04-18,2023-04-19,http://link
+Data Request In Review,Data Request In Review Description,2023-04-22,,
+Terminated,Terminated Description,,,
+Final Response,Final Response Description,,,
+Data Received,Data Received Description,,,
+RTI Data Is Live,RTI Data Is Live Description,,,

data/RTI/Lanka Fabric Ltd/2026-03-19/1/request.csv

@@ -0,0 +1,2 @@
+Title,Description,Source,Sender,Receiver Institution,Receiver Position
+Title 1,Description 1,http://sourcelink,LDF,Lanka Fabric Ltd,Minister

data/RTI/Lanka Fabric Ltd/2026-03-19/1/status.csv

@@ -0,0 +1,13 @@
+Status,Description,Entry Time,Exit Time,Source
+Request Identification,Request Identification Description,2022-09-14,2022-09-15,http://link
+Request Drafted,Request Drafted Description,2022-10-01,2022-10-02,http://link
+Draft Review,Draft Review Description,2022-10-03,2022-10-05,http://link
+Ready To Send,Ready To Send Description,2023-04-13,2023-04-21,http://link
+Request Sent,Request Sent Description,2023-04-14,2023-04-15,http://link
+Initial Request In Review,Initial Request In Review Description,2023-04-17,2023-04-18,http://link
+Initial Response,Initial Response Description,2023-04-18,2023-04-19,http://link
+Data Request In Review,Data Request In Review Description,2023-04-22,,
+Terminated,Terminated Description,,,
+Final Response,Final Response Description,,,
+Data Received,Data Received Description,,,
+RTI Data Is Live,RTI Data Is Live Description,,,

data/RTI/Minister of defence/2022-03-19/1/request.csv

@@ -0,0 +1,2 @@
+Title,Description,Source,Sender,Receiver Institution,Receiver Position
+Title 2,Description 2,http://sourcelink,LDF,Minister of defence,Secretary

data/RTI/Minister of defence/2022-03-19/1/status.csv

@@ -0,0 +1,13 @@
+Status,Description,Entry Time,Exit Time,Source
+Request Identification,Request Identification Description,2022-09-14,2022-09-15,http://link
+Request Drafted,Request Drafted Description,2022-10-01,2022-10-02,http://link
+Draft Review,Draft Review Description,2022-10-03,2022-10-05,http://link
+Ready To Send,Ready To Send Description,2023-04-13,2023-04-21,http://link
+Request Sent,Request Sent Description,2023-04-14,2023-04-15,http://link
+Initial Request In Review,Initial Request In Review Description,2023-04-17,2023-04-18,http://link
+Initial Response,Initial Response Description,2023-04-18,2023-04-19,http://link
+Data Request In Review,Data Request In Review Description,2023-04-22,,
+Terminated,Terminated Description,,,
+Final Response,Final Response Description,,,
+Data Received,Data Received Description,,,
+RTI Data Is Live,RTI Data Is Live Description,,,

data/RTI/Minister of defence/2022-03-19/2/request.csv

@@ -0,0 +1,2 @@
+Title,Description,Source,Sender,Receiver Institution,Receiver Position
+Title 3,Description 3,http://sourcelink,LDF,Minister of defence,Secretary

data/RTI/Minister of defence/2022-03-19/2/status.csv

@@ -0,0 +1,13 @@
+Status,Description,Entry Time,Exit Time,Source
+Request Identification,Request Identification Description,2022-09-14,2022-09-15,http://link
+Request Drafted,Request Drafted Description,2022-10-01,2022-10-02,http://link
+Draft Review,Draft Review Description,2022-10-03,2022-10-05,http://link
+Ready To Send,Ready To Send Description,2023-04-13,2023-04-21,http://link
+Request Sent,Request Sent Description,2023-04-14,2023-04-15,http://link
+Initial Request In Review,Initial Request In Review Description,2023-04-17,2023-04-18,http://link
+Initial Response,Initial Response Description,2023-04-18,2023-04-19,http://link
+Data Request In Review,Data Request In Review Description,2023-04-22,,
+Terminated,Terminated Description,,,
+Final Response,Final Response Description,,,
+Data Received,Data Received Description,,,
+RTI Data Is Live,RTI Data Is Live Description,,,

data/RTI/Minister of defence/2026-03-20/1/request.csv

@@ -0,0 +1,2 @@
+Title,Description,Source,Sender,Receiver Institution,Receiver Position
+Title 4,Description 4,http://sourcelink,LDF,Minister of defence,Minister

data/RTI/Minister of defence/2026-03-20/1/status.csv

@@ -0,0 +1,13 @@
+Status,Description,Entry Time,Exit Time,Source
+Request Identification,Request Identification Description,2022-09-14,2022-09-15,http://link
+Request Drafted,Request Drafted Description,2022-10-01,2022-10-02,http://link
+Draft Review,Draft Review Description,2022-10-03,2022-10-05,http://link
+Ready To Send,Ready To Send Description,2023-04-13,2023-04-21,http://link
+Request Sent,Request Sent Description,2023-04-14,2023-04-15,http://link
+Initial Request In Review,Initial Request In Review Description,2023-04-17,2023-04-18,http://link
+Initial Response,Initial Response Description,2023-04-18,2023-04-19,http://link
+Data Request In Review,Data Request In Review Description,2023-04-22,,
+Terminated,Terminated Description,,,
+Final Response,Final Response Description,,,
+Data Received,Data Received Description,,,
+RTI Data Is Live,RTI Data Is Live Description,,,

ingestion/.env.example

@@ -0,0 +1,2 @@
+export INGESTION_URL="http://localhost:8080"
+export READ_URL="http://localhost:8081"

ingestion/README.md

@@ -0,0 +1,25 @@
+## How to run the service
+
+#### source env
+```
+    source .env
+```
+
+#### run the tests
+```
+    go test ./tests/
+```
+
+#### run the scripts
+```
+    go run ./cmd/app/main.go --data "../data/RTI" -v
+```
+
+-------- or --------
+
+#### run the test run file
+```
+    source .env
+    chmod +X test_run.sh
+    ./test_run.sh
+```

ingestion/cmd/app/main.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"encoding/csv"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/LDFLK/RTI-Tracker/ingestion/internals/client"
+	"github.com/LDFLK/RTI-Tracker/ingestion/internals/core"
+	"github.com/LDFLK/RTI-Tracker/ingestion/internals/models"
+	"github.com/LDFLK/RTI-Tracker/ingestion/internals/ports"
+	"github.com/LDFLK/RTI-Tracker/ingestion/internals/utils"
+)
+
+func main() {
+
+	dataDir := flag.String("data", "", "Path to data directory containing csv files")
+
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n\n", os.Args[0])
+	}
+
+	flag.Parse()
+
+	// access environment variables
+	ingestionUrl := os.Getenv("INGESTION_URL")
+	if ingestionUrl == "" {
+		log.Fatal("Ingestion Service URL Required")
+	}
+
+	readUrl := os.Getenv("READ_URL")
+	if readUrl == "" {
+		log.Fatal("Read Service URL Required")
+	}
+
+	// Initialize services
+	apiClient := client.ApiClient(ingestionUrl, readUrl)
+	ingestionService := ports.NewIngestionService(*apiClient)
+	readService := ports.NewReadService(*apiClient)
+	s := core.NewRTIService(ingestionService, readService)
+
+	// validate flags
+	if *dataDir == "" {
+		fmt.Fprintf(os.Stderr, "Error: Data directory path is required\n\n")
+		os.Exit(1)
+	}
+
+	var requestErrors []string
+	var rtiRequestCount int
+
+	err := filepath.WalkDir(*dataDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			fmt.Printf("Error: %s\n\n", err)
+			return err
+		}
+
+		if d.IsDir() {
+			fmt.Printf("Directory: %s\n", path)
+		} else {
+			fmt.Printf("File: %s\n", path)
+			log.Println("Processing file...")
+
+			fileName := filepath.Base(path)
+			fileDir := filepath.Dir(path)
+			splittedDir := strings.Split(fileDir, string(filepath.Separator))
+
+			// access the date from the folder structure
+			date := splittedDir[len(splittedDir)-2]
+			index := splittedDir[len(splittedDir)-1]
+			dateISO, err := utils.DateToISO(date)
+
+			if err != nil {
+				log.Printf("failed to parse date %v", err)
+				return nil
+			}
+
+			if fileName == "status.csv" {
+				// attribute insertion process for status
+			} else {
+				// node creation process and attribute insertion for request
+				// open the file
+				f, err := os.Open(path)
+				if err != nil {
+					log.Printf("Failed to open file: %s , %v", path, err)
+					return nil
+				}
+
+				defer f.Close()
+
+				r := csv.NewReader(f)
+
+				// read the first line of the csv first to skip the first line
+				if _, err := r.Read(); err != nil {
+					fmt.Printf("Error accessing fields in csv: %s", err)
+				}
+
+				// access data starting from the second row in the csv
+				for {
+					record, err := r.Read()
+
+					if err == io.EOF {
+						break
+					}
+
+					if err != nil {
+						log.Println("Err reading records in the file")
+					}
+
+					title := record[0]
+					description := record[1]
+					source := record[2]
+					sender := record[3]
+					receiverInstitution := record[4]
+					receiverPosition := record[5]
+
+					// field data to the RTIRequest
+					entity := &models.RTIRequest{
+						Title:               title,
+						Description:         description,
+						Source:              source,
+						Sender:              sender,
+						ReceiverInstitution: receiverInstitution,
+						ReceiverPosition:    receiverPosition,
+						Created:             dateISO,
+						Index:               index,
+					}
+
+					_, err = s.InsertRTIRequest(entity)
+
+					if err != nil {
+						log.Printf("[main] RTI Insertion failed %s", err)
+						requestErrors = append(requestErrors, err.Error())
+						continue
+					}
+
+					rtiRequestCount++
+
+				}
+
+			}
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		log.Fatal(err)
+		os.Exit(0)
+	}
+
+	log.Printf("Processed %d RTI requests successfully (noted %d errors).", rtiRequestCount, len(requestErrors))
+
+}

ingestion/go.mod

@@ -0,0 +1,9 @@
+module github.com/LDFLK/RTI-Tracker/ingestion
+
+go 1.24.3
+
+require (
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
+)

ingestion/go.sum

@@ -0,0 +1,6 @@
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
+github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=

ingestion/internals/client/client.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/hashicorp/go-retryablehttp"
+)
+
+// API Client struct
+type Client struct {
+	IngestionURL string
+	ReadURL      string
+	HttpClient   *retryablehttp.Client
+}
+
+// Custom Retry Check
+func customRetryCheck(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if resp != nil {
+		switch resp.StatusCode {
+		case http.StatusBadRequest, http.StatusNotFound, http.StatusConflict:
+			return false, nil
+		}
+	}
+
+	return retryablehttp.DefaultRetryPolicy(ctx, resp, err)
+}
+
+// API Client
+func ApiClient(ingesUrl string, reUrl string) *Client {
+	retryClient := retryablehttp.NewClient()
+	retryClient.RetryMax = 10
+	retryClient.RetryWaitMin = 1 * time.Second
+	retryClient.RetryWaitMax = 6 * time.Second
+	retryClient.CheckRetry = customRetryCheck
+	retryClient.Logger = nil
+
+	return &Client{
+		IngestionURL: ingesUrl,
+		ReadURL:      reUrl,
+		HttpClient:   retryClient,
+	}
+}