Feature/backtesting 52 week high investment strategy

.gitignore

@@ -159,4 +159,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-*.db
+*.db
+src/config.json

src/BackTesting.py

@@ -1,21 +1,104 @@
 from classes.RunStrategy import *
+import time, random, os, tempfile, shutil
 
-class BackTesting():
+class BackTesting(RunStrategy):
     isBackTesting = True
-    __runStrategyObj = None
     __testConfigs = dict()
     __result = list()
+    __doWait = True
+    __startDate = None
+    __endDate = None
+    __useCache = False
+    __stockData = StockData(isBackTesting=True)
 
     def __init__(self, strategyName, testConfigs=None) -> None:
         self.__testConfigs = testConfigs
         iniBalance = 0.0
         if 'initialBalance' in testConfigs:
-            iniBalance = testConfigs['initialBalance']
-        self.__runStrategyObj = RunStrategy(strategyName, iniBalance)
+            iniBalance = self.__testConfigs['initialBalance']
+        if 'startDate' in testConfigs:
+            self.__startDate = testConfigs['startDate']
+        if 'endDate' in testConfigs:
+            self.__endDate = testConfigs['endDate']
+        if 'marketCap' in testConfigs:
+            self._marketCap = testConfigs['marketCap']
+        if 'stockList' in testConfigs:
+            self._stocksList = testConfigs['stockList']
+        if 'noWait' in testConfigs:
+            self.__doWait = False
+        if 'useCache' in testConfigs:
+            self.__useCache = testConfigs['useCache']
+        if not self.__useCache:
+            if os.path.exists(tempfile.gettempdir()+'/AlgoTradingBackTesting'):
+                shutil.rmtree(tempfile.gettempdir()+'/AlgoTradingBackTesting')
+            os.mkdir(tempfile.gettempdir()+'/AlgoTradingBackTesting')
+        else:
+            self.__doWait = False
+        super().__init__(strategyName, iniBalance, testConfigs)
+        self.isLive = False
 
-    def run(self):
-        '''@TODO: Implement this method'''
-        pass
+    def __selectStocks(self):
+        return self.__stockData.getStocksFromChartInk("(+{cash}+(+market+cap+>=+"+str(self._marketCap)+"+)+)+")
 
+    def __fetchStockData(self, stockList):
+        currentTime = int(time.time())
+        stockPriceTimeline = {}
+        stockListData = stockList['data']
+        count =1
+        maxDataLen = 0
+        timeLine=set()
+        maxDataLenStock = None
+        for i in stockListData:
+            print(str(count)+"/"+str(stockList['recordsFiltered'])+" Getting data for "+i['nsecode']+"...")
+            data = self.__stockData.getStockDataFromApi(i['nsecode'], currentTime, '1D', {"cacheEnabled": True})
+            if len(data) == 0:
+                print("No data found. Skipping...")
+                continue
+            if len(data) > maxDataLen:
+                maxDataLenStock = i['nsecode']
+            maxDataLen = max(maxDataLen, len(data))
+            startTime = data[0][0]
+            print("Got data from "+str(time.strftime('%d-%m-%Y', time.localtime(startTime))))
+            stockPriceTimeline[i['nsecode']] = {'name': i['name'], 'nsecode': i['nsecode'], 'data': data}          
+            print()
+            count+=1
+            if self.__doWait:
+                time.sleep(random.random())
+        return (stockPriceTimeline,maxDataLen)
+
+
+    def run(self):
+        self._strategy.cleanData()
+        stockList = self.__selectStocks()
+        stockListData = stockList['data']
+        print("Found total "+str(stockList['recordsFiltered'])+" records")
+        print("Getting data for each stock... \n")
+        stockPriceTimeline, maxDataLen = self.__fetchStockData(stockList)       
+        for i in range(52*5, maxDataLen):
+            print("Running for loop "+str(i)+".. Total left: "+str(maxDataLen-1-i))
+            strategyData = []
+            week10thLow = {}
+            currentTime = None
+            for j in stockPriceTimeline.items():
+                if len(j[1]['data']) < maxDataLen:
+                    padLen = maxDataLen - len(j[1]['data'])
+                    stockPriceTimeline[j[0]]['data'] = [[0,0,0,0,0,0]]*padLen + stockPriceTimeline[j[0]]['data']
+                    j[1]['data']=stockPriceTimeline[j[0]]['data']
+                currentTime = j[1]['data'][i][0]
+                if currentTime == 0:
+                    continue
+                stockData = {'nsecode': j[1]['nsecode'], 'name': j[1]['name'], 'close': j[1]['data'][i][4], 'currentTime': currentTime}
+                close52WeekHigh = StrategyUtils.get52WeekHigh({'stockData': j[1]['data'][:i+1]})
+                if j[1]['data'][i][4] >= close52WeekHigh and close52WeekHigh>0:
+                    strategyData.append(stockData)
+            try:
+                self._strategy.triggerSellStock({'currentTime': currentTime})
+                self._strategy.configParams({'stockList': strategyData})
+                self._strategy.triggerBuyStock({'stockList': strategyData})
+            except Exception as e:
+                print(e)
+                raise e
+
+
     def getResult(self) -> list:
-        return self.__result
+        return self._strategy.get_transactionHistory()

src/__init__.py

@@ -0,0 +1,2 @@
+import pathlib, os, sys
+sys.path.append(os.path.abspath(pathlib.Path(__file__).parent.absolute()))

src/classes/Config.py

@@ -1,23 +1,27 @@
-import json, sqlite3, os, sys, requests
+import json, sqlite3, os, sys, requests, copy
 
 class Config():
     '''
     Config class to set configuration files, secret file, initializing database and brocker session.
     \nInstantiation of this class is restricted.
     '''
-    __configFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'\\'+"config.json"
-    __secretFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'\\'+"secret.json"
+    __configFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'/'+"config.json"
+    __secretFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'/'+"secret.json"
     __stockDB = 'stocks.db'
     __backtestingDB = 'backtestingStocks.db'
     __configValues = {}
     __secretValues = {}
-    __dbCursor = list()
+    __dbConnections = list()
     __brokerSession = requests.session()
 
     # To restrict object creation, self is removed from parameter
     def __init__() -> None:
         raise Exception("Object cannot be created for Config class")
 
+    @staticmethod
+    def setupEnvironment(isBacktesting = False):
+        return (copy.deepcopy(Config.getBrokerSession()), Config.getDBCursors(isBacktesting))
+
     @staticmethod
     def setConfigFiles(configFilePath=None, secretFilePath=None):
         if configFilePath is not None:
@@ -29,12 +33,18 @@ def setConfigFiles(configFilePath=None, secretFilePath=None):
     @staticmethod
     def getConfigValues(index):
         Config.__loadConfigurations()
-        return Config.__configValues[index]
+        if index not in Config.__configValues:
+            return None
+        else:
+            return Config.__configValues[index]
 
     @staticmethod
-    def getDBCursors():
+    def getDBCursors(isBackTesting = False):
         Config.__initDBConnections()
-        return Config.__dbCursor
+        if not isBackTesting:
+            return (Config.__dbConnections[0].cursor(), Config.__dbConnections[0])
+        else:
+            return (Config.__dbConnections[1].cursor(), Config.__dbConnections[1])
 
     @staticmethod
     def __loadConfigurations():
@@ -48,7 +58,7 @@ def __loadConfigurations():
     def __initDBConnections():
         stockDB = sqlite3.connect(Config.__stockDB)
         backTestDB = sqlite3.connect(Config.__backtestingDB)
-        Config.__dbCursor = [stockDB.cursor, backTestDB.cursor]
+        Config.__dbConnections = [stockDB, backTestDB]
 
     @staticmethod
     def getBrokerSession():

src/classes/DBConnector.py

@@ -0,0 +1,139 @@
+from Config import *
+import time
+
+class DBConnector():
+    _selectedDBCursor = None
+    _selectedDB = None
+    _isBackTesting = False
+
+
+    def __init__(self, isBackTesting = False) -> None:
+        self._isBackTesting = isBackTesting
+        self.selectDB(isBackTesting)
+
+    def selectDB(self, isBackTesting = False):
+        dbObjects = Config.getDBCursors(isBackTesting)
+        DBConnector._selectedDBCursor = dbObjects[0]
+        DBConnector._selectedDB = dbObjects[1]
+        self.__initDB()
+
+    def resetDB(self, skipConfirm = False):
+        if not skipConfirm:
+            confirm = input('Are you sure you want to reset DB? (Y): ')
+            if confirm == 'Y':
+                pass
+            else:
+                print('DB Reset aborted..')
+                return
+        cursor = self._selectedDBCursor
+        cursor.execute('''DROP TABLE stocksTxn''')
+        cursor.execute('''DROP TABLE strategy''')
+        cursor.execute('''DROP TABLE stocks''')
+        self._selectedDB.commit()
+        self.__initDB()  
+
+    def __initDB(self):
+        cursor = self._selectedDBCursor
+        cursor.execute('''CREATE TABLE IF NOT EXISTS stocks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name string,
+        nseCode string UNIQUE
+        );''')
+
+        cursor.execute('''CREATE TABLE IF NOT EXISTS config (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            key string UNIQUE,
+            value string
+        );''')
+
+        cursor.execute('''DROP TABLE IF EXISTS strategy;''')
+        cursor.execute('''
+        CREATE TABLE strategy (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name string UNIQUE,
+        enable bool DEFAULT True
+        );''')
+        cursor.execute('''
+        INSERT INTO strategy (name) VALUES 
+        ('None'), ('SampleStrategy'), ('WeekHigh52');
+        ''')
+
+        cursor.execute('''CREATE TABLE IF NOT EXISTS stocksTxn (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        stockId INTEGER NOT NULL,
+        txnDate DATETIME DEFAULT CURRENT_TIMESTAMP,
+        txnTime string DEFAULT "",
+        holdingStatus varchar(1),
+        price float,
+        quantity integer,
+        strategy integer DEFAULT 1,
+        stopLoss float,
+        stopLossPercent float,
+        targetPrice float,
+        targetPercent float,
+        status string,
+        enable Boolean DEFAULT TRUE,
+        syncStatus Boolean DEFAULT FALSE,
+        FOREIGN KEY(stockId) REFERENCES stocks(id),
+  		FOREIGN KEY(strategy) REFERENCES strategy(id),
+  		UNIQUE(stockId, txnTime, holdingStatus, price, quantity, strategy, status)
+        )''')
+
+        self._selectedDB.commit()
+
+    def addNewTxn(self, stockName, nseCode, holdingStatus:str, price:float, quantity:int, stopLossPercent:float, targetPercent:float, status:str, strategy=0, txnTime=None, stopLoss:float = None, targetPrice:float= None, syncStatus=0):
+        stockId = self.addStocks(stockName, nseCode)
+        if stopLoss is None:
+            stopLoss = price - (stopLossPercent/100)*price
+        if targetPrice is None:
+            targetPrice = price + (targetPercent/100)*price
+        self.addStocksTxn(stockId, holdingStatus, price, quantity, stopLoss, stopLossPercent, targetPrice, targetPercent, status, strategy, txnTime, syncStatus)
+
+
+    def addStocks(self, name:str, nseCode:str):
+        query = "INSERT OR IGNORE INTO stocks (name, nseCode) VALUES (?, ?)"
+        response = self._selectedDBCursor.execute(query, (name, nseCode,))    
+        self._selectedDB.commit() 
+        stockID = self.selectDataFromTable('stocks', columnName='id', whereClause='nseCode="'+nseCode+'"')[0][0]
+        return stockID
+
+    def addStocksTxn(self, stockId:int, holdingStatus:str, price:float, quantity:int, stopLoss: float, stopLossPercent:float, targetPrice:float, targetPercent:float, status:str, strategy=0, txnTime=None, syncStatus=0):
+        if txnTime is None:
+            txnTime = int(time.time())        
+        query = '''INSERT OR IGNORE INTO stocksTxn (stockId,holdingStatus,price,quantity,strategy,stopLoss,stopLossPercent,targetPrice,targetPercent,status,txnTime,syncStatus) 
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
+        '''
+        self._selectedDBCursor.execute(query, (stockId,holdingStatus,price,quantity,strategy,stopLoss,stopLossPercent,targetPrice,targetPercent,status, txnTime, syncStatus))
+        self._selectedDB.commit()
+
+    def selectDataFromTable(self, table, whereClause=None, columnName='*'):
+        query = '''SELECT {0} FROM {1} '''
+        if whereClause is not None and whereClause!='':
+            query += '''WHERE '''+whereClause
+        query = query.format(columnName, table)
+        self._selectedDBCursor.execute(query)
+        resp = self._selectedDBCursor.fetchall()
+        return resp
+
+    def executeRawQuery(self, query, isSelectQuery=False):
+        queryResponse = self._selectedDBCursor.execute(query)
+        if isSelectQuery:
+            queryResult = self._selectedDBCursor.fetchall()
+            return (queryResponse, queryResult)
+        else:
+            self._selectedDB.commit()
+        return (queryResponse)
+
+    def getConfig(self, key):
+        result = self.selectDataFromTable('config', 'key="'+key+'"', 'value')
+        if len(result) > 0:
+            return result[0][0]
+        else:
+            return None
+
+    def setConfig(self, key, value):
+        existingValue = self.getConfig(key)
+        if existingValue is not None:
+            self.executeRawQuery('UPDATE config SET value="'+str(value)+'" WHERE key="'+key+'"')
+        else:
+            self.executeRawQuery('INSERT INTO config (key, value) VALUES("'+key+'", "'+str(value)+'")')

src/classes/RunStrategy.py

@@ -1,22 +1,45 @@
-from classes.Strategy import *
+from Strategy import *
 
 class RunStrategy():
-    __strategy = None
+    isBackTesting = False
+    _strategy = None
     __stockData = list()
-    __startDate = None
-    __endDate = None
+    isLive = False                 # Set it to True if System is Live
 
     def __init__(self, strategyName, initialBalance, configParams=None, stockData=None, startDate=None, endDate=None):
-        self.__strategy = registeredStrategy[strategyName](strategyName, initialBalance)
         if configParams is not None:
-            self.__strategy.configParams(configParams)
+            if 'isBackTesting' in configParams:
+                self.isBackTesting = configParams['isBackTesting']
+            if 'noWait' in configParams:
+                self.__doWait = False
+            if 'isLive' in configParams:
+                self.isLive = configParams['isLive']
+            else:
+                self.isLive = False
         if stockData is not None:
             self.__stockData = stockData
-            self.__strategy._updateStockList(stockData)
-        if startDate is not None:
-            self.__startDate = startDate
-        if endDate is not None:
-            self.__endDate = endDate
+            self._strategy._updateStockList(stockData)
+        self._strategy = registeredStrategy[strategyName](initialBalance, self.isBackTesting)
+        self._strategy.configParams(configParams)
+
+    def syncTransactions(self, type='B'):
+        '''
+        @TODO Override this method in child class to sync Transactions with Broker
+        '''
+        dbConn = DBConnector(self.isBackTesting)
+        getPendingTxn = dbConn.executeRawQuery('''SELECT st.id, s.id, s.name, s.nseCode, st.price, st.quantity FROM stocksTxn as st INNER JOIN stocks as s WHERE st.stockId=s.id AND st.syncStatus=0 AND st.holdingStatus="'''+type+'"')
+        return getPendingTxn
+
+    def run(self):
+        self._strategy.selectStocks()
+        self._strategy.triggerBuyStock()
+        self.syncTransactions('B')
+        self._strategy.triggerSellStock()
+        self.syncTransactions('S')
+
+        txn = self._strategy.getOnHoldTransactions()
+
+        return txn            
 
     def generateReport(self):
         pass