diff --git a/.gitignore b/.gitignore index ba7454e..e91a3e9 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,5 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -*.db \ No newline at end of file +*.db +src/config.json diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/doc/UML/UML_Diagraim.drawio b/doc/UML/UML_Diagraim.drawio index 54da17e..40f1b41 100644 --- a/doc/UML/UML_Diagraim.drawio +++ b/doc/UML/UML_Diagraim.drawio @@ -1,4 +1,4 @@ - + @@ -57,7 +57,7 @@ - + @@ -86,7 +86,7 @@ - + @@ -103,7 +103,7 @@ - + @@ -149,7 +149,7 @@ - + @@ -169,11 +169,8 @@ - - - - + @@ -181,16 +178,17 @@ - + - - + + + @@ -220,14 +218,15 @@ - - + + + - + @@ -247,15 +246,45 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/UML/UML_Diagraim.svg b/doc/UML/UML_Diagraim.svg index 237c0c5..e15a7fa 100644 --- a/doc/UML/UML_Diagraim.svg +++ b/doc/UML/UML_Diagraim.svg @@ -1,4 +1,4 @@ -Config- configFilePath : String
 - secretFilePath : String
- stockDB : String
- backTestingDB : String
- configValues : Json
- secretValues : Json
- dbCursors : <dbCursor, dbCursor>
- brokerSession : request.session
- Config()
+ setConfigFiles(String <config.json>, String <secret.json>)
+ getConfigValues(): Json
+ getDBCursors(): <dbCursor, dbCursor>
- loadConfigurations(): void
- initDBConnections(): void
+ getBrokerSession(): Object
1
StockData
+ isBacktesting: boolean
 - config : Config
+ getInstance(Config = None) : StockData
- StockData(Config)
+ getStockData(symbol: string, startDate : String, endDate : String) : list
StrategyUtils
+ calculateSMA(days : int, data : List)
Strategy
+ isBacktesting: boolean
- strategyName : String
- initialBalance : float
- currentBalance : float
- stocksList : List<>
- transactionHistory : List<>
- additionalParams : Dict
# Strategy(name, initialBalance)
# setBalance(int)
# addProfit(int) : balance (int)
# addLoss(int) : balance (int)
# updateStockList(List<>): bool
+ configParameters() : pass
+ selectStocks() : pass
+ triggerBuyStock() : pass
+ triggerSellStock() : pass
Backtesting
isBackTesting = True
- runStrategy : RunStrategy
- testConfigs : List<Dict>
- result : List
+ Backtesting(strategy, testConfigs)
+ run()
+ getResult() : List
Use
Use
Use
SampleStrategy
+ SampleStrategy(name, initialBalance)
+ configParameters()
+ selectStocks() : List
+ triggerBuyStock() : int
+ triggerSellStock() : int
Extends
Use
RunStrategy
- strategy : Strategy
- stockData: List  = None
- startDate: Date
- endDate: Date
+ RunStrategy(name, initialBalance, configParams, stockData=None, startDate=None, endDate=None)
+ generateReport() : void
Use
 \ No newline at end of file +Config- configFilePath : String
 - secretFilePath : String
- stockDB : String
- backTestingDB : String
- configValues : Json
- secretValues : Json
- dbCursors : <dbCursor, dbCursor>
- brokerSession : request.session
- Config()
+ setConfigFiles(String <config.json>, String <secret.json>)
+ getConfigValues(): Json
+ getDBCursors(): <dbCursor, dbCursor>
- loadConfigurations(): void
- initDBConnections(): void
+ getBrokerSession(): Object
1
StockData
+ isBacktesting: boolean
 - config : Config
+ getInstance(Config = None) : StockData
- StockData(Config)
+ getStockData(symbol: string, startDate : String, endDate : String) : list
StrategyUtils
+ calculateSMA(days : int, data : List)
Strategy
+ isBacktesting: boolean
- strategyName : String
- initialBalance : float
- currentBalance : float
- stocksList : List<>
- transactionHistory : List<>
- additionalParams : Dict
# Strategy(name, initialBalance)
# setBalance(int)
# addProfit(int) : balance (int)
# addLoss(int) : balance (int)
# updateStockList(List<>): bool
+ configParameters() : pass
+ selectStocks() : pass
+ triggerBuyStock() : pass
+ triggerSellStock() : pass
Backtesting
isBackTesting = True
- runStrategy : RunStrategy
- testConfigs : List<Dict>
- result : List
+ Backtesting(strategy, testConfigs)
+ getResult() : List
Use
Use
Use
SampleStrategy
+ SampleStrategy(name, initialBalance)
+ configParameters()
+ selectStocks() : List
+ triggerBuyStock() : int
+ triggerSellStock() : int
Extends
Extends
RunStrategy
- strategy : Strategy
- stockData: List  = None
- startDate: Date
- endDate: Date
+ RunStrategy(name, initialBalance, configParams, stockData=None, startDate=None, endDate=None)
+ run()
+ generateReport() : void
Use
 \ No newline at end of file diff --git a/src/BackTesting.py b/src/BackTesting.py index 7a5303b..1a0bfa0 100644 --- a/src/BackTesting.py +++ b/src/BackTesting.py @@ -1,21 +1,104 @@ from classes.RunStrategy import * +import time, random, os, tempfile, shutil -class BackTesting(): +class BackTesting(RunStrategy): isBackTesting = True - __runStrategyObj = None __testConfigs = dict() __result = list() + __doWait = True + __startDate = None + __endDate = None + __useCache = False + __stockData = StockData(isBackTesting=True) def __init__(self, strategyName, testConfigs=None) -> None: self.__testConfigs = testConfigs iniBalance = 0.0 if 'initialBalance' in testConfigs: - iniBalance = testConfigs['initialBalance'] - self.__runStrategyObj = RunStrategy(strategyName, iniBalance) + iniBalance = self.__testConfigs['initialBalance'] + if 'startDate' in testConfigs: + self.__startDate = testConfigs['startDate'] + if 'endDate' in testConfigs: + self.__endDate = testConfigs['endDate'] + if 'marketCap' in testConfigs: + self._marketCap = testConfigs['marketCap'] + if 'stockList' in testConfigs: + self._stocksList = testConfigs['stockList'] + if 'noWait' in testConfigs: + self.__doWait = False + if 'useCache' in testConfigs: + self.__useCache = testConfigs['useCache'] + if not self.__useCache: + if os.path.exists(tempfile.gettempdir()+'/AlgoTradingBackTesting'): + shutil.rmtree(tempfile.gettempdir()+'/AlgoTradingBackTesting') + os.mkdir(tempfile.gettempdir()+'/AlgoTradingBackTesting') + else: + self.__doWait = False + super().__init__(strategyName, iniBalance, testConfigs) + self.isLive = False - def run(self): - '''@TODO: Implement this method''' - pass + def __selectStocks(self): + return self.__stockData.getStocksFromChartInk("(+{cash}+(+market+cap+>=+"+str(self._marketCap)+"+)+)+") + def __fetchStockData(self, stockList): + currentTime = int(time.time()) + stockPriceTimeline = {} + stockListData = stockList['data'] + count =1 + maxDataLen = 0 + timeLine=set() + maxDataLenStock = None + for i in stockListData: + print(str(count)+"/"+str(stockList['recordsFiltered'])+" Getting data for "+i['nsecode']+"...") + data = self.__stockData.getStockDataFromApi(i['nsecode'], currentTime, '1D', {"cacheEnabled": True}) + if len(data) == 0: + print("No data found. Skipping...") + continue + if len(data) > maxDataLen: + maxDataLenStock = i['nsecode'] + maxDataLen = max(maxDataLen, len(data)) + startTime = data[0][0] + print("Got data from "+str(time.strftime('%d-%m-%Y', time.localtime(startTime)))) + stockPriceTimeline[i['nsecode']] = {'name': i['name'], 'nsecode': i['nsecode'], 'data': data} + print() + count+=1 + if self.__doWait: + time.sleep(random.random()) + return (stockPriceTimeline,maxDataLen) + + + def run(self): + self._strategy.cleanData() + stockList = self.__selectStocks() + stockListData = stockList['data'] + print("Found total "+str(stockList['recordsFiltered'])+" records") + print("Getting data for each stock... \n") + stockPriceTimeline, maxDataLen = self.__fetchStockData(stockList) + for i in range(52*5, maxDataLen): + print("Running for loop "+str(i)+".. Total left: "+str(maxDataLen-1-i)) + strategyData = [] + week10thLow = {} + currentTime = None + for j in stockPriceTimeline.items(): + if len(j[1]['data']) < maxDataLen: + padLen = maxDataLen - len(j[1]['data']) + stockPriceTimeline[j[0]]['data'] = [[0,0,0,0,0,0]]*padLen + stockPriceTimeline[j[0]]['data'] + j[1]['data']=stockPriceTimeline[j[0]]['data'] + currentTime = j[1]['data'][i][0] + if currentTime == 0: + continue + stockData = {'nsecode': j[1]['nsecode'], 'name': j[1]['name'], 'close': j[1]['data'][i][4], 'currentTime': currentTime} + close52WeekHigh = StrategyUtils.get52WeekHigh({'stockData': j[1]['data'][:i+1]}) + if j[1]['data'][i][4] >= close52WeekHigh and close52WeekHigh>0: + strategyData.append(stockData) + try: + self._strategy.triggerSellStock({'currentTime': currentTime}) + self._strategy.configParams({'stockList': strategyData}) + self._strategy.triggerBuyStock({'stockList': strategyData}) + except Exception as e: + print(e) + raise e + + def getResult(self) -> list: - return self.__result \ No newline at end of file + return self._strategy.get_transactionHistory() \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e95fff8 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,2 @@ +import pathlib, os, sys +sys.path.append(os.path.abspath(pathlib.Path(__file__).parent.absolute())) \ No newline at end of file diff --git a/src/classes/Config.py b/src/classes/Config.py index 533d5b8..1165019 100644 --- a/src/classes/Config.py +++ b/src/classes/Config.py @@ -1,23 +1,27 @@ -import json, sqlite3, os, sys, requests +import json, sqlite3, os, sys, requests, copy class Config(): ''' Config class to set configuration files, secret file, initializing database and brocker session. \nInstantiation of this class is restricted. ''' - __configFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'\\'+"config.json" - __secretFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'\\'+"secret.json" + __configFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'/'+"config.json" + __secretFilePath=os.path.dirname(os.path.abspath(sys.argv[0]))+'/'+"secret.json" __stockDB = 'stocks.db' __backtestingDB = 'backtestingStocks.db' __configValues = {} __secretValues = {} - __dbCursor = list() + __dbConnections = list() __brokerSession = requests.session() # To restrict object creation, self is removed from parameter def __init__() -> None: raise Exception("Object cannot be created for Config class") + @staticmethod + def setupEnvironment(isBacktesting = False): + return (copy.deepcopy(Config.getBrokerSession()), Config.getDBCursors(isBacktesting)) + @staticmethod def setConfigFiles(configFilePath=None, secretFilePath=None): if configFilePath is not None: @@ -29,12 +33,18 @@ def setConfigFiles(configFilePath=None, secretFilePath=None): @staticmethod def getConfigValues(index): Config.__loadConfigurations() - return Config.__configValues[index] + if index not in Config.__configValues: + return None + else: + return Config.__configValues[index] @staticmethod - def getDBCursors(): + def getDBCursors(isBackTesting = False): Config.__initDBConnections() - return Config.__dbCursor + if not isBackTesting: + return (Config.__dbConnections[0].cursor(), Config.__dbConnections[0]) + else: + return (Config.__dbConnections[1].cursor(), Config.__dbConnections[1]) @staticmethod def __loadConfigurations(): @@ -48,7 +58,7 @@ def __loadConfigurations(): def __initDBConnections(): stockDB = sqlite3.connect(Config.__stockDB) backTestDB = sqlite3.connect(Config.__backtestingDB) - Config.__dbCursor = [stockDB.cursor, backTestDB.cursor] + Config.__dbConnections = [stockDB, backTestDB] @staticmethod def getBrokerSession(): diff --git a/src/classes/DBConnector.py b/src/classes/DBConnector.py new file mode 100644 index 0000000..e250731 --- /dev/null +++ b/src/classes/DBConnector.py @@ -0,0 +1,139 @@ +from Config import * +import time + +class DBConnector(): + _selectedDBCursor = None + _selectedDB = None + _isBackTesting = False + + + def __init__(self, isBackTesting = False) -> None: + self._isBackTesting = isBackTesting + self.selectDB(isBackTesting) + + def selectDB(self, isBackTesting = False): + dbObjects = Config.getDBCursors(isBackTesting) + DBConnector._selectedDBCursor = dbObjects[0] + DBConnector._selectedDB = dbObjects[1] + self.__initDB() + + def resetDB(self, skipConfirm = False): + if not skipConfirm: + confirm = input('Are you sure you want to reset DB? (Y): ') + if confirm == 'Y': + pass + else: + print('DB Reset aborted..') + return + cursor = self._selectedDBCursor + cursor.execute('''DROP TABLE stocksTxn''') + cursor.execute('''DROP TABLE strategy''') + cursor.execute('''DROP TABLE stocks''') + self._selectedDB.commit() + self.__initDB() + + def __initDB(self): + cursor = self._selectedDBCursor + cursor.execute('''CREATE TABLE IF NOT EXISTS stocks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name string, + nseCode string UNIQUE + );''') + + cursor.execute('''CREATE TABLE IF NOT EXISTS config ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key string UNIQUE, + value string + );''') + + cursor.execute('''DROP TABLE IF EXISTS strategy;''') + cursor.execute(''' + CREATE TABLE strategy ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name string UNIQUE, + enable bool DEFAULT True + );''') + cursor.execute(''' + INSERT INTO strategy (name) VALUES + ('None'), ('SampleStrategy'), ('WeekHigh52'); + ''') + + cursor.execute('''CREATE TABLE IF NOT EXISTS stocksTxn ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + stockId INTEGER NOT NULL, + txnDate DATETIME DEFAULT CURRENT_TIMESTAMP, + txnTime string DEFAULT "", + holdingStatus varchar(1), + price float, + quantity integer, + strategy integer DEFAULT 1, + stopLoss float, + stopLossPercent float, + targetPrice float, + targetPercent float, + status string, + enable Boolean DEFAULT TRUE, + syncStatus Boolean DEFAULT FALSE, + FOREIGN KEY(stockId) REFERENCES stocks(id), + FOREIGN KEY(strategy) REFERENCES strategy(id), + UNIQUE(stockId, txnTime, holdingStatus, price, quantity, strategy, status) + )''') + + self._selectedDB.commit() + + def addNewTxn(self, stockName, nseCode, holdingStatus:str, price:float, quantity:int, stopLossPercent:float, targetPercent:float, status:str, strategy=0, txnTime=None, stopLoss:float = None, targetPrice:float= None, syncStatus=0): + stockId = self.addStocks(stockName, nseCode) + if stopLoss is None: + stopLoss = price - (stopLossPercent/100)*price + if targetPrice is None: + targetPrice = price + (targetPercent/100)*price + self.addStocksTxn(stockId, holdingStatus, price, quantity, stopLoss, stopLossPercent, targetPrice, targetPercent, status, strategy, txnTime, syncStatus) + + + def addStocks(self, name:str, nseCode:str): + query = "INSERT OR IGNORE INTO stocks (name, nseCode) VALUES (?, ?)" + response = self._selectedDBCursor.execute(query, (name, nseCode,)) + self._selectedDB.commit() + stockID = self.selectDataFromTable('stocks', columnName='id', whereClause='nseCode="'+nseCode+'"')[0][0] + return stockID + + def addStocksTxn(self, stockId:int, holdingStatus:str, price:float, quantity:int, stopLoss: float, stopLossPercent:float, targetPrice:float, targetPercent:float, status:str, strategy=0, txnTime=None, syncStatus=0): + if txnTime is None: + txnTime = int(time.time()) + query = '''INSERT OR IGNORE INTO stocksTxn (stockId,holdingStatus,price,quantity,strategy,stopLoss,stopLossPercent,targetPrice,targetPercent,status,txnTime,syncStatus) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + ''' + self._selectedDBCursor.execute(query, (stockId,holdingStatus,price,quantity,strategy,stopLoss,stopLossPercent,targetPrice,targetPercent,status, txnTime, syncStatus)) + self._selectedDB.commit() + + def selectDataFromTable(self, table, whereClause=None, columnName='*'): + query = '''SELECT {0} FROM {1} ''' + if whereClause is not None and whereClause!='': + query += '''WHERE '''+whereClause + query = query.format(columnName, table) + self._selectedDBCursor.execute(query) + resp = self._selectedDBCursor.fetchall() + return resp + + def executeRawQuery(self, query, isSelectQuery=False): + queryResponse = self._selectedDBCursor.execute(query) + if isSelectQuery: + queryResult = self._selectedDBCursor.fetchall() + return (queryResponse, queryResult) + else: + self._selectedDB.commit() + return (queryResponse) + + def getConfig(self, key): + result = self.selectDataFromTable('config', 'key="'+key+'"', 'value') + if len(result) > 0: + return result[0][0] + else: + return None + + def setConfig(self, key, value): + existingValue = self.getConfig(key) + if existingValue is not None: + self.executeRawQuery('UPDATE config SET value="'+str(value)+'" WHERE key="'+key+'"') + else: + self.executeRawQuery('INSERT INTO config (key, value) VALUES("'+key+'", "'+str(value)+'")') \ No newline at end of file diff --git a/src/classes/RunStrategy.py b/src/classes/RunStrategy.py index eeba4e3..f9efaa2 100644 --- a/src/classes/RunStrategy.py +++ b/src/classes/RunStrategy.py @@ -1,22 +1,45 @@ -from classes.Strategy import * +from Strategy import * class RunStrategy(): - __strategy = None + isBackTesting = False + _strategy = None __stockData = list() - __startDate = None - __endDate = None + isLive = False # Set it to True if System is Live def __init__(self, strategyName, initialBalance, configParams=None, stockData=None, startDate=None, endDate=None): - self.__strategy = registeredStrategy[strategyName](strategyName, initialBalance) if configParams is not None: - self.__strategy.configParams(configParams) + if 'isBackTesting' in configParams: + self.isBackTesting = configParams['isBackTesting'] + if 'noWait' in configParams: + self.__doWait = False + if 'isLive' in configParams: + self.isLive = configParams['isLive'] + else: + self.isLive = False if stockData is not None: self.__stockData = stockData - self.__strategy._updateStockList(stockData) - if startDate is not None: - self.__startDate = startDate - if endDate is not None: - self.__endDate = endDate + self._strategy._updateStockList(stockData) + self._strategy = registeredStrategy[strategyName](initialBalance, self.isBackTesting) + self._strategy.configParams(configParams) + + def syncTransactions(self, type='B'): + ''' + @TODO Override this method in child class to sync Transactions with Broker + ''' + dbConn = DBConnector(self.isBackTesting) + getPendingTxn = dbConn.executeRawQuery('''SELECT st.id, s.id, s.name, s.nseCode, st.price, st.quantity FROM stocksTxn as st INNER JOIN stocks as s WHERE st.stockId=s.id AND st.syncStatus=0 AND st.holdingStatus="'''+type+'"') + return getPendingTxn + + def run(self): + self._strategy.selectStocks() + self._strategy.triggerBuyStock() + self.syncTransactions('B') + self._strategy.triggerSellStock() + self.syncTransactions('S') + + txn = self._strategy.getOnHoldTransactions() + + return txn def generateReport(self): pass \ No newline at end of file diff --git a/src/classes/StockData.py b/src/classes/StockData.py index 0f7a84c..b36bb20 100644 --- a/src/classes/StockData.py +++ b/src/classes/StockData.py @@ -1,24 +1,27 @@ import requests, datetime, json -from classes.Config import Config +from Config import Config +from DBConnector import * +import tempfile, os, time, pickle class StockData(): - isBackTesting=False + __isBackTesting=False __config = list() __requestHeader = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0', 'Cache-Control' : 'no-cache, must-revalidate' } + _s = requests.session() - - def __new__(cls, config=None): + def __new__(cls, config=None, isBackTesting=False): if not hasattr(cls, 'instance'): cls.instance = super(StockData, cls).__new__(cls) return cls.instance - def __init__(self, config=None): + def __init__(self, config=None, isBackTesting=False): #Overwritting config values if config is not None: self.__config = config + self.__isBackTesting = isBackTesting def __splitDateRange(self, startDate, endDate, intervalDays=70) -> list: ''' @@ -70,17 +73,17 @@ def getStockDataFromNSE(self, symbol, startDate, endDate) -> list: return stockData - def getStockDataFromApi(self, *args): - ''' - Fetch Stock CSV data from internet, based on url format provided in config file - :args arguments to pass in url - :returns dictionary data {'s': 'ok', 't': [..], 'o': [..], 'h': [..], 'l': [..], 'c':[..], 'v':[..]} -> status, time, open, high, low, close, volume - ''' + def __getStockDataFromApi_Live(self, *args): global s - url = Config.getConfigValues('stockDataURL').format(*args) + symbol = args[0][0] + to = args[0][1] + resolution = args[0][2] + url = Config.getConfigValues('stockDataURL').format(symbol, to, resolution) #convert to required format: - data = json.loads(s.get(url, headers=self.__requestHeader).content.decode()) + data = json.loads(self._s.get(url, headers=self.__requestHeader).content.decode()) stockTimeLine = [] + if data['s'] != 'ok': + return stockTimeLine for i in range(0, len(data['t'])): lst = [] lst.append(data['t'][i]) @@ -91,5 +94,77 @@ def getStockDataFromApi(self, *args): lst.append(data['v'][i]) stockTimeLine.append(lst) return stockTimeLine - - \ No newline at end of file + + def __getNearestIndex(self, data, indexVal): + for i in range(0, len(data)): + if data[i] == indexVal: + return i + elif data[i] > indexVal: + return i-1 + return len(data)-1 + + def getStockDataFromApi(self, *args): + ''' + Fetch Stock CSV data from internet, based on url format provided in config file + :args arguments to pass in url + :returns dictionary data {'s': 'ok', 't': [..], 'o': [..], 'h': [..], 'l': [..], 'c':[..], 'v':[..]} -> status, time, open, high, low, close, volume + ''' + global s + cacheEnabled = True + if 'cacheEnabled' in args[-1]: + cacheEnabled = args[-1]['cacheEnabled'] + + cacheFound = False + + if cacheEnabled: + symbol = args[0] + dirPath = tempfile.gettempdir()+'/AlgoTradingBackTesting/' + cacheFile = dirPath+symbol+'_'+args[2]+'.pk' + if os.path.exists(cacheFile): + cFile = open(cacheFile, 'rb') + cacheFound = True + else: + if not os.path.exists(dirPath): + os.mkdir(dirPath) + cFile = open(cacheFile, 'wb+') + if cacheFound: + result = pickle.load(cFile) + timeInd = self.__getNearestIndex(result['index'], args[1]) + finalResult = result['data'][:timeInd+1] + return finalResult + else: + result = self.__getStockDataFromApi_Live(args[0], int(time.time()), args[2]) + index = [] + for i in result: + index.append(i[0]) + dumpData = { + 'data': result, + 'index': index + } + timeInd = self.__getNearestIndex(index, args[1]) + finalResult = result[:timeInd+1] + pickle.dump(dumpData, cFile) + cFile.close() + return finalResult + else: + return self.__getStockDataFromApi_Live(args) + + + def getStocksFromChartInk(self, scan_clause): + tmpSession = requests.session() + res = tmpSession.get(Config.getConfigValues('scanner')).content.decode() + csrf_str = 'csrf-token" content="' + ind = res.find(csrf_str)+len(csrf_str) + csrf_token = '' + for i in range(ind, ind+100): + if res[i] == '"': + break + csrf_token += res[i] + + screener_head = { + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'X-Csrf-Token': csrf_token + } + urlString = 'process?scan_clause='+scan_clause + res = json.loads(tmpSession.post(Config.getConfigValues('scanningURL').format(urlString,), headers=screener_head).content.decode()) + return res diff --git a/src/classes/Strategy.py b/src/classes/Strategy.py index b986e78..eb698ff 100644 --- a/src/classes/Strategy.py +++ b/src/classes/Strategy.py @@ -1,36 +1,68 @@ +from StockData import * +from StrategyUtils import * +import copy, time + class Strategy(): isBackTesting = False - __strategyName = '' - __initialBalance = 0.0 - __currentBalance = 0.0 - __stocksList = list() - __transactionHistory = list() - __PLHistory = list() - __aditionalParameters = dict() + _strategyName = '' + _initialBalance = 0.0 + _currentBalance = 0.0 + _stocksList = list() + _transactionHistory = list() # {'name', 'nseCode', 'qnt', 'price', 'total', 'txnType', 'date'} + _PLHistory = list() + _aditionalParameters = dict() + _dbConnector = None + _configParams = None + _strategyId = -1 def __init__(self, strategyName:str, initialBalance) -> None: - self.__strategyName = strategyName - self.__initialBalance = float(initialBalance) + self._strategyName = strategyName + self._initialBalance = float(initialBalance) + self._currentBalance = self._initialBalance + + def get_StrategyName(self): + return self._strategyName + + def get_strategyId(self): + return self._strategyId + + def get_currentBalance(self): + return self._currentBalance + + def get_stocksList(self): + return copy.deepcopy(self._stocksList) + + def get_transactionHistory(self): + return copy.deepcopy(self._transactionHistory) + + def get_PLHistory(self): + return copy.deepcopy(self._PLHistory) def _setBalance(self, newBalance: float): - self.__currentBalance = newBalance + self._currentBalance = newBalance def _addProfit(self, profitAmount: float): - self.__currentBalance += profitAmount - self.__PLHistory.append({"amount": profitAmount, "type": "P"}) + self._currentBalance += profitAmount + self._PLHistory.append({"amount": profitAmount, "type": "P"}) def _addLoss(self, lossAmount:float): - self.__currentBalance -= lossAmount - self.__PLHistory.append({"amount": lossAmount, "type": "L"}) + self._currentBalance -= lossAmount + self._PLHistory.append({"amount": lossAmount, "type": "L"}) def _updateStockList(self, newStockList: list): - self.__stocksList = newStockList + ''' + Provide stock list manually to the strategy. + ''' + self._stocksList = newStockList def configParams(self, *args): raise Exception("Method not implemented, Please overwrite this method") pass def selectStocks(self, *args): + ''' + This method will fetch and select the stocks from APIs based on the conditions met for the strategy. + ''' raise Exception("Method not implemented, Please overwrite this method") pass @@ -41,23 +73,210 @@ def triggerBuyStock(self, *args): def triggerSellStock(self, *args): raise Exception("Method not implemented, Please overwrite this method") pass + + def cleanData(self): + if not self.isBackTesting: + inp = input('You are trying to delete production data. Are you sure? (Type `sure`)') + if inp == 'sure': + self._dbConnector.executeRawQuery('''DELETE FROM stocksTxn WHERE strategy='''+str(self._strategyId), False) + else: + print('Skipping deletion') + return True + else: + self._dbConnector.executeRawQuery('''DELETE FROM stocksTxn WHERE strategy='''+str(self._strategyId), False) + class SampleStrategy(Strategy): - def __init__(self, strategyName: str, initialBalance: float) -> None: - super().__init__(strategyName, initialBalance) + _strategyName = 'SampleStrategy' + + def __init__(self, initialBalance: float) -> None: + super().__init__(self._strategyName, initialBalance) def configParams(self, *args): pass def selectStocks(self, *args): - pass - + largeCapStocks = StockData.getStocksFromChartInk("(+{cash}+(+market+cap+>+10000+)+)+") + self._stocksList = largeCapStocks['data'] + def triggerBuyStock(self, *args): - pass - + # Buy stock is price is less than 1000 + for i in self.__stocksList: + nseCode = i['nseCode'] + name = i['name'] + price = i['close'] + qnt = 1 + if float(price) < 1000 and self._currentBalance>0: + cost = qnt*price + self._transactionHistory.append({'name': name, 'nseCode': nseCode, 'qnt': qnt, 'price': price, 'total': cost, 'txnType': 'B'}) + self._currentBalance -= cost + def triggerSellStock(self, *args): + # Sell stocks already in hold if profit min 5% and set SL at 2%, no shorting + loop = False + if 'keepRunning' in args: + loop = True pass -registeredStrategy = {'SampleStrategy' : SampleStrategy} \ No newline at end of file +class WeekHigh52(Strategy): + ''' + Config Params: + stopLossPercent:float + targetPercent:float + marketCap:int (in Crores) + ''' + + _strategyName = 'WeekHigh52' + _stockData = StockData() + _stopLossPercent = 30 + _targetPercent = 60 + _marketCap = 1000 + _strategyId = 3 + _txnDBEntry = dict() + _maxBuyLimitPerStock = 1000 + + def __init__(self, initialBalance: float, isBackTesting=False) -> None: + super().__init__(self._strategyName, initialBalance) + self.isBackTesting = isBackTesting + self._dbConnector = DBConnector(isBackTesting) + WeekHigh52._strategyId = self._dbConnector.selectDataFromTable('strategy', 'name="'+WeekHigh52._strategyName+'"')[0][0] + + def configParams(self, *args): + configParam = args[0] + if configParam is None: + self._configParams = configParam + return + # print(configParam[0]) + if self._configParams is None: + self._configParams = configParam + else: + self._configParams.update(configParam) + if 'stopLossPercent' in configParam: + self._stopLossPercent = configParam['stopLossPercent'] + if 'targetPercent' in configParam: + self._targetPercent = configParam['targetPercent'] + if 'marketCap' in configParam: + self._marketCap = configParam['marketCap'] + if 'stockList' in configParam: + self._stocksList = configParam['stockList'] + if 'maxBuyLimitPerStock' in configParam: + self._maxBuyLimitPerStock = configParam['maxBuyLimitPerStock'] + + def selectStocks(self, *args): + weekhigh52 = self._stockData.getStocksFromChartInk("(+{cash}+(+latest+close+>+1+day+ago+max(+260+,+latest+high+)+and+market+cap+>=+"+str(self._marketCap)+"+)+)+") + self._stocksList = weekhigh52['data'] + + def getOnHoldTransactions(self): + return self._dbConnector.executeRawQuery('''SELECT st.*, s.* FROM stocksTxn st INNER JOIN stocks s ON st.stockId=s.id WHERE st.status='Hold' AND st.strategy='''+str(self._strategyId), True)[1] + + def __addTransactionToDB(self, stockName, nseCode, holdingStatus, price, qnt, stopLoss, targetPrice, status, txnTime=None): + self._dbConnector.addStocks(stockName, nseCode) + stockID = self._dbConnector.selectDataFromTable('stocks', 'nseCode = "'+nseCode+'"')[0][0] + self._dbConnector.addStocksTxn(stockID, holdingStatus, price, qnt, stopLoss, self._stopLossPercent, targetPrice, self._targetPercent, status, self._strategyId, txnTime) + txnId = self._dbConnector.selectDataFromTable('stocksTxn', 'stockId = "'+str(stockID)+'" AND holdingStatus="'+holdingStatus+'" AND price="'+str(price)+'" AND quantity="'+str(qnt)+'" AND strategy="'+str(self._strategyId)+'" AND status="'+status+'"')[0][0] + return txnId + + def triggerBuyStock(self, *args): + data = None + if len(args)>0: + data = args[0] + if data is not None and 'stockList' in data and 'currentTime' in data['stockList']: + currentTime = data['stockList']['currentTime'] + else: + currentTime = int(time.time()) + # Buy stock is price is less than 1000 + for i in self._stocksList: + nseCode = i['nsecode'] + name = i['name'] + price = i['close'] + qnt = 1 + if 'quantity' in i: + qnt = i['quantity'] + if 'currentTime' in i: + currentTime = i['currentTime'] + if float(price) < self._maxBuyLimitPerStock and self._currentBalance>0 and float(price) <= self._currentBalance: + print("Purchasing Stock...") + if self._configParams is not None and 'buyMaxLimit' in self._configParams and self._configParams['buyMaxLimit']: + qnt = min(int(self._maxBuyLimitPerStock/price), int(self._currentBalance/price)) + cost = qnt*price + self._currentBalance -= cost + stopLoss = price - (self._stopLossPercent/100)*price + targetPrice = price + (self._targetPercent/100)*price + txnId = self.__addTransactionToDB(name, nseCode, 'B', price, qnt, stopLoss, targetPrice, 'Hold', currentTime) + self._transactionHistory.append({'txnId': txnId, 'name': name, 'nseCode': nseCode, 'qnt': qnt, 'price': price, 'total': cost, 'txnType': 'B', 'status': 'Hold', 'date': currentTime}) + if nseCode not in self._txnDBEntry: + self._txnDBEntry[nseCode] = list() + self._txnDBEntry[nseCode].append(txnId) + elif self._currentBalance <= 0: + print("BALANCE IS OVER.. Cannot BUY MORE STOCKS") + + + def triggerSellStock(self, *args): + # Sell stocks already in hold if profit min 5% and set SL at 2%, no shorting + ''' + Conditions: + 1. Run scanner at 3 PM + 2. Cost > 52 week high + 3. Fixed stop loss = 30% + 4. Fix Target = 60% + 5. Trailing stop loss = last 10th week low + ''' + data = None + if len(args)>0: + data = args[0] + strategyTxn = self._dbConnector.executeRawQuery(''' + SELECT st.id, st.stockId, st.txnDate, st.txnTime, st.holdingStatus, + st.price, st.quantity, st.strategy, st.stopLoss, st.stopLossPercent, + st.targetPrice, st.targetPercent, st.status, st.enable, st.syncStatus, + s.name, s.nseCode + FROM stocksTxn st INNER JOIN stocks s ON st.stockId=s.id WHERE st.status='Hold' AND st.holdingStatus='B' AND st.strategy='''+str(self._strategyId), True)[1] + for i in strategyTxn: + txnId = i[0] + nseCode = i[16] + + name = i[15] + price = i[5] + qnt = i[6] + currentStopLoss = i[8] + stopLossPercent = i[9] + targetPrice = i[10] + targetPricePercent = i[11] + if data is not None and 'currentTime' in data: + currentTime = data['currentTime'] + else: + currentTime = int(time.time()) + get10WeekLow = StrategyUtils.getLow(nseCode, 10, '1W', currentTime, False) + + # Input from arguments + if data is not None and 'latestClose' in data: + latestClose = float(data['latestClose'][nseCode]) + else: + latestCloseData = self._stockData.getStockDataFromApi(nseCode, currentTime, '1D', *args) + if len(latestCloseData) ==0: + latestClose = 0 + print("Error in getting latest close value from API..") + raise Exception("Error in getting latest close value from API..") + #Error + else: + latestClose = latestCloseData[-1][4] + + if currentStopLoss >= latestClose: + print("Stop Loss already triggered") + self._currentBalance+=(currentStopLoss*qnt) + self._transactionHistory.append({'txnId': txnId, 'name': name, 'nseCode': nseCode, 'qnt': qnt, 'price': currentStopLoss, 'total': currentStopLoss*qnt, 'txnType': 'S', 'status': 'Hold', 'date': currentTime}) + self._dbConnector.executeRawQuery("UPDATE stocksTxn SET holdingStatus='S', status='Executed' WHERE id="+str(txnId)) + + # Check if stock is increasing, update the stopLoss as well. + newCalculatedStopLoss = latestClose - (stopLossPercent/100)*latestClose + newStopLoss = max(get10WeekLow, max(currentStopLoss, newCalculatedStopLoss)) + + # Check if stock broke the target Price as well, set stopLoss to 1% less than Target Price + if latestClose > targetPrice: + newStopLoss = max(newStopLoss, targetPrice - 0.01*targetPrice) + + if currentStopLoss!= newStopLoss: + self._dbConnector.executeRawQuery("UPDATE stocksTxn SET stopLoss ='"+str(newStopLoss)+"', status='Hold', syncStatus='False' WHERE id="+str(txnId)) + +registeredStrategy = {'SampleStrategy' : SampleStrategy, 'WeekHigh52' : WeekHigh52} \ No newline at end of file diff --git a/src/classes/StrategyUtils.py b/src/classes/StrategyUtils.py index c8fed01..53a2058 100644 --- a/src/classes/StrategyUtils.py +++ b/src/classes/StrategyUtils.py @@ -1,3 +1,5 @@ +from StockData import * +import time class StrategyUtils(): @staticmethod @@ -9,4 +11,60 @@ def calculateMA(pricelist, days=50): break sum += i[4] count += 1 - return round(sum / days, 2) \ No newline at end of file + return round(sum / days, 2) + + @staticmethod + def getLow(symbol:str, interval:int, intervalType:str, currentTime=None, cacheEnabled = True): + sd = StockData() + if currentTime is None: + currentTime = int(time.time()) + stockData = sd.getStockDataFromApi(symbol, currentTime, intervalType, {'cacheEnabled': cacheEnabled}) + if len(stockData) < interval: + return -1 + lowData = stockData[-1][3] + for i in range(1, interval+1): + lowData = min(lowData, stockData[-i][3]) + return lowData + + @staticmethod + def get52WeekHigh(*args): + data = args[0] + minExpectedLength = 52*5 + symbol = None + interval = None + intervalType = '1D' + highData = -1 + stockData = None + parsingType = 'CloseData' + + if 'symbol' in data: + symbol = data['symbol'] + if 'interval' in data: + interval = data['interval'] + if 'intervalType' in data: + intervalType= data['intervalType'] + if 'stockData' in data: + stockData = data['stockData'] + parsingType = 'FullData' + if 'stockCloseData' in data: + stockData = data['stockCloseData'] + parsingType = 'CloseData' + + if symbol is not None and interval is not None and intervalType is not None: + sd = StockData() + currentTime = int(time.time()) + stockData = sd.getStockDataFromApi(symbol, currentTime, intervalType) + highData = stockData[-interval][3] + elif stockData is not None and parsingType=='FullData': + for i in stockData[len(stockData)-minExpectedLength:]: + if i[0] == 0: + return -1 + # picking index as 2 for highest of the day + highData = max(highData, i[2]) + elif stockData is not None and parsingType=='CloseData': + for i in stockData[len(stockData)-minExpectedLength:]: + if i[0] == 0: + return -1 + highData = max(highData, i) + return highData + \ No newline at end of file diff --git a/src/classes/__init__.py b/src/classes/__init__.py index e69de29..e95fff8 100644 --- a/src/classes/__init__.py +++ b/src/classes/__init__.py @@ -0,0 +1,2 @@ +import pathlib, os, sys +sys.path.append(os.path.abspath(pathlib.Path(__file__).parent.absolute())) \ No newline at end of file diff --git a/src/config.json b/src/config.json index a89f233..41b9350 100644 --- a/src/config.json +++ b/src/config.json @@ -1,4 +1,5 @@ { "Comments": "This file contains sample config data. Please replace it with your own APIs/data to use", - "stockDataURL": "https://testAPI.com/testingStockData?symbol={0}&resolution=1D&from={1}&to={2}" + "stockDataURL": "https://testAPI.com/testingStockData?symbol={0}&from={1}&to={2}&resolution=1D", + "scanningURL": "https://chartink.com/screener/{0}" } \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..88c3a49 --- /dev/null +++ b/src/main.py @@ -0,0 +1,105 @@ +from BackTesting import * +from classes.Config import * +import time + + +def configEnvironment(): + Config.setupEnvironment() + + +def generateCsv(result, fileName = 'Report.csv'): + finalTxn = {} + totalProfit = 0 + totalTransactions = 0 + for i in result: + txnId = i['txnId'] + if txnId not in finalTxn.keys(): + finalTxn[txnId]={} + newDict = { + 'name': i['name'], + 'nseCode': i['nseCode'], + 'qnt': i['qnt'], + } + if 'name' not in finalTxn[txnId]: + finalTxn[txnId] = newDict + if i['txnType'] == 'B': + finalTxn[txnId]['buyingPrice'] = i['price'] + finalTxn[txnId]['totalBuyingPrice'] = i['total'] + finalTxn[txnId]['buyingDate'] = time.strftime('%d-%m-%Y', time.gmtime(i['date'])) + else: + finalTxn[txnId]['sellingPrice'] = i['price'] + finalTxn[txnId]['totalSellingPrice'] = i['total'] + finalTxn[txnId]['sellingDate'] = time.strftime('%d-%m-%Y', time.gmtime(i['date'])) + if 'buyingDate' in finalTxn[txnId] and 'sellingDate' in finalTxn[txnId]: + finalTxn[txnId]['profit'] = i['total'] - finalTxn[txnId]['totalBuyingPrice'] + + file = open(fileName, 'w+') + file.write('txnId, name, nseCode, quantity, buyingPrice, totalBuyingPrice, buyingDate, sellingPrice, totalSellingPrice, sellingDate, Profit\n') + for i in finalTxn.items(): + data = i[1] + buyingPrice = 0 + totalBuyingPrice = 0 + buyingDate = 'NA' + sellingPrice = 0 + totalSellingPrice = 0 + sellingDate = 'NA' + if 'buyingPrice' in data: + buyingPrice = data['buyingPrice'] + if 'totalBuyingPrice' in data: + totalBuyingPrice= data['totalBuyingPrice'] + if 'buyingDate' in data: + buyingDate= data['buyingDate'] + if 'sellingPrice' in data: + sellingPrice= data['sellingPrice'] + if 'totalSellingPrice' in data: + totalSellingPrice= data['totalSellingPrice'] + if 'sellingDate' in data: + sellingDate= data['sellingDate'] + profit = totalSellingPrice-totalBuyingPrice + totalProfit+= profit + totalTransactions+=1 + file.write(str(i[0])+','+data['name']+','+data['nseCode']+','+str(data['qnt'])+','+str(buyingPrice)+','+str(totalBuyingPrice)+','+buyingDate+','+str(sellingPrice)+','+str(totalSellingPrice)+','+sellingDate+','+str(profit)+'\n') + file.write("\n\nTotalProfit,"+str(totalProfit)+"\nTotalTransactions,"+str(totalTransactions)) + file.close() + return(totalProfit, totalTransactions) + + +configEnvironment() +config = { + "initialBalance": [10000, 20000, 50000, 100000], + 'stopLossPercent':[40, 30, 20], + 'targetPercent':[80, 60, 40], + 'marketCap':10000, + 'isBackTesting': True, + 'maxBuyLimitPerStock': 3000, + 'useCache': True, + 'buyMaxLimit': True +} +# bk = BackTesting(strategyName="WeekHigh52", testConfigs=config) +reportSummary = {} +for i in config['initialBalance']: + for jj in range(0, len(config['stopLossPercent'])): + j = config['stopLossPercent'][jj] + k = config['targetPercent'][jj] + print("\n"*5) + print("Running for initialBalance: "+str(i)+" stopLossPercent: "+str(j)+" targetPercent: "+str(k)+"\n") + newConfig = { + "initialBalance": i, + 'stopLossPercent':j, + 'targetPercent':k, + 'marketCap':10000, + 'isBackTesting': True, + 'maxBuyLimitPerStock': 3000, + 'useCache': True, + 'buyMaxLimit': True + } + bk = BackTesting(strategyName="WeekHigh52", testConfigs=newConfig) + bk.run() + profit, txnCount = generateCsv(bk.getResult(), "Report_Bal"+str(i)+"_SL"+str(j)+"_TP"+str(k)+".csv") + reportSummary[str(i)+'_'+str(j)+'_'+str(k)] = { + 'profit': profit, + 'transactionCount': txnCount + } + + print(reportSummary) + \ No newline at end of file