diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml new file mode 100644 index 00000000..620ad45d --- /dev/null +++ b/.github/workflows/continuous_integration.yml @@ -0,0 +1,77 @@ +name: Continuous Integration Pipeline + +# Triggering the workflow on push or pull request to the main branch +on: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + setup-and-build: + name: Continuous Integration + runs-on: ubuntu-latest + + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: common_assess + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping --silent" + --health-interval=10s + --health-timeout=5s + --health-retries=3 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4.8.0 + with: + python-version: 3.10.11 + + - name: Check versions + run: | + python --version + pip --version + + - name: Upgrade pip + run: | + python -m pip install --upgrade pip + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pymysql + + - name: Linter Check + run: | + pip install pylint + pylint $(git ls-files '*.py') + + - name: Configure Test Enviroment + run: | + pip install pytest + pytest + + - name: Wait for MySQL to be ready + run: | + for i in {1..30}; do + mysqladmin ping -h 127.0.0.1 --silent && break + echo "Waiting for MySQL..." && sleep 2 + done + + - name: Apply Database Migrations + run: | + python app/database.py + + - name: Run Application + run: | + nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload & diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d369a361 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.11 + +# Set the working directory +WORKDIR /app + +# Copy the requirements file +COPY app/requirements.txt . + +# Install the dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy rest of the app code +COPY . /app + +# Expose the port +EXPOSE 8000 + +# Command to run application +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/app/__init__.py b/app/__init__.py index e69de29b..12fe4bd1 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -0,0 +1,3 @@ +""" +Package for client-related functionality (CRUD operations, schemas, etc.). +""" diff --git a/app/clients/clients_create_tables.sql b/app/clients/clients_create_tables.sql new file mode 100644 index 00000000..73a3cf3e --- /dev/null +++ b/app/clients/clients_create_tables.sql @@ -0,0 +1,34 @@ +CREATE SCHEMA IF NOT EXISTS Clients; + +USE Clients; + +DROP TABLE IF EXISTS Persons; + +CREATE TABLE Persons ( + person_id BIGINT AUTO_INCREMENT, + age INT, + gender VARCHAR(255), + work_experience INT, + canada_workex INT, + dep_num INT, + canada_born VARCHAR(255), + citizen_status VARCHAR(255), + level_of_schooling VARCHAR(255), + fluent_english VARCHAR(255), + reading_english_scale INT, + speaking_english_scale INT, + writing_english_scale INT, + numeracy_scale INT, + computer_scale INT, + transportation_bool VARCHAR(255), + caregiver_bool VARCHAR(255), + housing VARCHAR(255), + income_source VARCHAR(255), + felony_bool VARCHAR(255), + attending_school VARCHAR(255), + currently_employed VARCHAR(255), + substance_use VARCHAR(255), + time_unemployed INT, + need_mental_health_support_bool VARCHAR(255), + CONSTRAINT pk_Persons_PersonId PRIMARY KEY (person_id) +); \ No newline at end of file diff --git a/app/clients/crud.py b/app/clients/crud.py new file mode 100644 index 00000000..671e8ec9 --- /dev/null +++ b/app/clients/crud.py @@ -0,0 +1,51 @@ +"""Module providing CRUD functions.""" + +from sqlalchemy.orm import Session +from .models import Client +from .schema import ClientCreate, ClientUpdate + + +def create_client(db: Session, client: ClientCreate): + '''Creates a new client record in the database''' + db_client = Client(**client.dict()) + db.add(db_client) + db.commit() + db.refresh(db_client) + return db_client + + +def get_client(db: Session, client_id: int): + '''Retrieves a client record by client_id''' + return db.query(Client).filter(Client.id == client_id).first() + + +def get_clients(db: Session, skip: int = 0, limit: int = 10): + '''Retrieves a list of client records with custom formatting. + + Args: + skip (int): Number of client records to skip from start of list + limit (int): Maximum number of client records to return + ''' + return db.query(Client).offset(skip).limit(limit).all() + + +def update_client(db: Session, client_id: int, client: ClientUpdate): + '''Updates an existing client record with only the fields provided in client''' + db_client = db.query(Client).filter(Client.id == client_id).first() + if not db_client: + # Raise an error if the client is not found + raise ValueError(f"Client with ID {client_id} not found.") + # Update the client with the provided data + for key, value in client.dict(exclude_unset=True).items(): + setattr(db_client, key, value) + db.commit() + db.refresh(db_client) + return db_client + +def delete_client(db: Session, client_id: int): + '''Deletes a client record by client_id''' + db_client = db.query(Client).filter(Client.id == client_id).first() + if db_client: + db.delete(db_client) + db.commit() + return db_client diff --git a/app/clients/data_source.sql b/app/clients/data_source.sql new file mode 100644 index 00000000..f7f17615 --- /dev/null +++ b/app/clients/data_source.sql @@ -0,0 +1,62 @@ +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for clients +-- ---------------------------- +DROP TABLE IF EXISTS `clients`; +CREATE TABLE `clients` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL, + `age` int NULL DEFAULT NULL, + `gender` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL, + `work_experience` int NULL DEFAULT NULL, + `canada_workex` int NULL DEFAULT NULL, + `dep_num` int NULL DEFAULT NULL, + `canada_born` tinyint(1) NULL DEFAULT NULL, + `citizen_status` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL, + `level_of_schooling` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL, + `fluent_english` tinyint(1) NULL DEFAULT NULL, + `reading_english_scale` int NULL DEFAULT NULL, + `speaking_english_scale` int NULL DEFAULT NULL, + `writing_english_scale` int NULL DEFAULT NULL, + `numeracy_scale` int NULL DEFAULT NULL, + `computer_scale` int NULL DEFAULT NULL, + `transportation_bool` tinyint(1) NULL DEFAULT NULL, + `caregiver_bool` tinyint(1) NULL DEFAULT NULL, + `housing` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL, + `income_source` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL, + `felony_bool` tinyint(1) NULL DEFAULT NULL, + `attending_school` tinyint(1) NULL DEFAULT NULL, + `currently_employed` tinyint(1) NULL DEFAULT NULL, + `substance_use` tinyint(1) NULL DEFAULT NULL, + `time_unemployed` int NULL DEFAULT NULL, + `need_mental_health_support_bool` tinyint(1) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `ix_clients_name`(`name`) USING BTREE, + INDEX `ix_clients_id`(`id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of clients +-- ---------------------------- +INSERT INTO `clients` VALUES (1, 'Joe', 25, '1', 6, 2, '1', '1', '1', '5', '1', 7, 6, 5, 4, 6, '1', '1', '3', '4', '1', '1', '1', '1', 12, '1'); +INSERT INTO `clients` VALUES (2, 'Mike', 32, '2', 10, 5, '2', '0', '0', '8', '0', 8, 9, 7, 6, 7, '0', '0', '2', '2', '0', '0', '0', '0', 8, '0'); +INSERT INTO `clients` VALUES (3, 'Leo', 24, '1', 2, 1, '3', '1', '1', '3', '1', 6, 4, 6, 5, 4, '1', '1', '5', '3', '1', '1', '1', '1', 10, '1'); +INSERT INTO `clients` VALUES (4, 'Alice', 29, '2', 7, 3, '0', '0', '0', '4', '0', 7, 8, 5, 7, 9, '0', '0', '6', '5', '0', '0', '0', '0', 6, '0'); +INSERT INTO `clients` VALUES (5, 'Clara', 31, '1', 5, 4, '4', '1', '1', '7', '1', 6, 5, 4, 8, 6, '1', '1', '4', '7', '1', '1', '1', '1', 15, '1'); +INSERT INTO `clients` VALUES (6, 'Ella', 22, '2', 1, 0, '1', '0', '0', '2', '0', 7, 8, 6, 7, 5, '0', '0', '2', '1', '0', '0', '0', '0', 5, '0'); +INSERT INTO `clients` VALUES (7, 'Bella', 34, '1', 12, 6, '3', '1', '1', '10', '1', 9, 8, 10, 9, 9, '1', '1', '7', '6', '1', '1', '1', '1', 0, '1'); +INSERT INTO `clients` VALUES (8, 'Diana', 26, '2', 4, 2, '2', '1', '1', '3', '1', 5, 7, 6, 5, 8, '1', '1', '5', '3', '1', '1', '1', '1', 9, '1'); +INSERT INTO `clients` VALUES (9, 'Fiona', 30, '1', 8, 3, '0', '0', '0', '6', '0', 10, 9, 8, 6, 7, '0', '0', '3', '2', '0', '0', '0', '0', 3, '0'); +INSERT INTO `clients` VALUES (10, 'Grace', 27, '2', 6, 5, '1', '1', '1', '5', '1', 8, 6, 7, 8, 6, '1', '1', '1', '4', '1', '1', '1', '1', 4, '1'); +INSERT INTO `clients` VALUES (11, 'India', 25, '1', 3, 1, '2', '0', '0', '4', '0', 6, 5, 4, 6, 5, '0', '0', '3', '3', '0', '0', '0', '0', '0', 7, '0'); +INSERT INTO `clients` VALUES (12, 'Karen', 33, '2', 10, 7, '0', '1', '1', '9', '1', 9, 10, 8, 9, 10, '1', '1', '8', '7', '1', '1', '1', '1', 2, '1'); +INSERT INTO `clients` VALUES (13, 'Hannah', 20, '1', 1, 0, '1', '0', '0', '1', '0', 5, 3, 5, 4, 3, '0', '0', '2', '1', '0', '0', '0', '0', 14, '0'); +INSERT INTO `clients` VALUES (14, 'Julia', 36, '2', 11, 6, '2', '1', '1', '8', '1', 10, 9, 10, 8, 9, '1', '1', '5', '6', '1', '1', '1', '1', 1, '1'); +INSERT INTO `clients` VALUES (15, 'Luna', 29, '1', 5, 2, '4', '1', '1', '5', '1', 8, 7, 6, 6, 8, '1', '1', '7', '5', '1', '1', '1', '1', 11, '1'); +INSERT INTO `clients` VALUES (16, 'Mia', 23, '2', 2, 1, '1', '0', '0', '3', '0', 7, 6, 7, 5, 6, '0', '0', '4', '2', '0', '0', '0', '0', '0', 10, '0'); +INSERT INTO `clients` VALUES (17, 'Olivia', 35, '1', 8, 4, '3', '1', '1', '7', '1', 9, 10, 9, 8, 9, '1', '1', '3', '6', '1', '1', '1', '1', 6, '1'); +INSERT INTO `clients` VALUES (18, 'Qiana', 28, '2', 6, 2, '0', '0', '0', '6', '0', 6, 8, 5, 7, 7, '0', '0', '1', '4', '0', '0', '0', '0', 12, '0'); +INSERT INTO `clients` VALUES (19, 'Nora', 21, '1', 2, 0, '2', '1', '1', '2', '1', 5, 4, 6, 3, 5, '1', '1', '2', '2', '1', '1', '1', '1', 13, '1'); +INSERT INTO `clients` VALUES (20, 'Poppy', 24, '2', 3, 1, '0', '0', '0', '3', '0', 6, 5, 7, 6, 6, '0', '0', '3', '3', '0', '0', '0', '0', 16, '0'); diff --git a/app/clients/models.py b/app/clients/models.py new file mode 100644 index 00000000..16cd23e0 --- /dev/null +++ b/app/clients/models.py @@ -0,0 +1,43 @@ +'''Module providing functions for building the client model''' + +from sqlalchemy import Column, Integer, String, Boolean +from app.database import Base # Need to create database configuration module in app.database + + +# pylint: disable=too-few-public-methods +class Client(Base): + '''Class defining the client model''' + # Setting the table name for this model to 'clients' + __tablename__ = 'clients' + + # Primary Key column, unique identifier for each client record + id = Column(Integer, primary_key=True, index=True) + name = Column(String, index=True) + age = Column(Integer) + gender = Column(String) + work_experience = Column(Integer) + canada_workex = Column(Integer) + dep_num = Column(Integer) + canada_born = Column(Boolean) + citizen_status = Column(String) + level_of_schooling = Column(String) + fluent_english = Column(Boolean) + reading_english_scale = Column(Integer) + speaking_english_scale = Column(Integer) + writing_english_scale = Column(Integer) + numeracy_scale = Column(Integer) + computer_scale = Column(Integer) + transportation_bool = Column(Boolean) + caregiver_bool = Column(Boolean) + housing = Column(String) + income_source = Column(String) + felony_bool = Column(Boolean) + attending_school = Column(Boolean) + currently_employed = Column(Boolean) + substance_use = Column(Boolean) + time_unemployed = Column(Integer) + need_mental_health_support_bool = Column(Boolean) + + def __repr__(self): + '''Provide string representation of the Client instance''' + return f"" diff --git a/app/clients/router.py b/app/clients/router.py index f860c402..5ab3ac30 100644 --- a/app/clients/router.py +++ b/app/clients/router.py @@ -1,15 +1,61 @@ -from fastapi import APIRouter -from fastapi.responses import HTMLResponse +'''Model containing functions for router''' +from typing import List +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session from app.clients.service.logic import interpret_and_calculate -from app.clients.schema import PredictionInput +from app.clients.schema import PredictionInput, ClientCreate, ClientUpdate, Client +from app.clients.crud import create_client, get_client, get_clients, update_client, delete_client +from app.database import get_db # Need to create a get_db function to provide DB sessions + router = APIRouter(prefix="/clients", tags=["clients"]) @router.post("/predictions") async def predict(data: PredictionInput): + '''Prediction endpoint''' print("HERE") print(data.model_dump()) return interpret_and_calculate(data.model_dump()) +@router.post("/", response_model=Client) +async def create_new_client(client: ClientCreate, db: Session = Depends(get_db)): + '''Create a new client''' + return create_client(db=db, client=client) + + +@router.get("/{client_id}", response_model=Client) +async def read_client(client_id: int, db: Session = Depends(get_db)): + '''Get a client by client_id''' + client = get_client(db=db, client_id=client_id) + if client is None: + raise HTTPException(status_code=404, detail="Client not found") + return client + + +@router.get("/", response_model=List[Client]) +async def read_clients(skip: int = 0, limit: int = 10, db: Session = Depends(get_db)): + '''Retrieve clients with options to skip a number of results and limit max results shown''' + return get_clients(db=db, skip=skip, limit=limit) + + +@router.put("/{client_id}", response_model=Client) +async def update_existing_client( + client_id: int, + client: ClientUpdate, + db: Session = Depends(get_db)): + '''Update a client's information by client_id''' + db_client = update_client(db=db, client_id=client_id, client=client) + if db_client is None: + raise HTTPException(status_code=404, detail="Client not found") + return {f"Client with ID {client_id} is updated successfully!"} + + +@router.delete("/{client_id}", response_model=Client) +async def delete_existing_client(client_id: int, db: Session = Depends(get_db)): + '''Delete a client by client_id''' + db_client = delete_client(db=db, client_id=client_id) + if db_client is None: + raise HTTPException(status_code=404, detail="Client not found") + return {f"Client with ID {client_id} is deleted successfully!"} diff --git a/app/clients/schema.py b/app/clients/schema.py index 6b56ad98..e636dedc 100644 --- a/app/clients/schema.py +++ b/app/clients/schema.py @@ -1,6 +1,12 @@ +"""Model contains functions to create schema""" + +from typing import Optional from pydantic import BaseModel +# pylint: disable=too-few-public-methods class PredictionInput(BaseModel): + """Create schema for PredictionInput""" + age: int gender: str work_experience: int @@ -25,3 +31,77 @@ class PredictionInput(BaseModel): substance_use: str time_unemployed: int need_mental_health_support_bool: str + + +class ClientBase(BaseModel): + """Create schema for ClientBase""" + + name: str + age: int + gender: str + work_experience: int + canada_workex: int + dep_num: int + canada_born: bool + citizen_status: str + level_of_schooling: str + fluent_english: bool + reading_english_scale: int + speaking_english_scale: int + writing_english_scale: int + numeracy_scale: int + computer_scale: int + transportation_bool: bool + caregiver_bool: bool + housing: str + income_source: str + felony_bool: bool + attending_school: bool + currently_employed: bool + substance_use: bool + time_unemployed: int + need_mental_health_support_bool: bool + + +class ClientCreate(ClientBase): + """Schema for creating a new client""" + + +class ClientUpdate(BaseModel): + """Schema for updating client information with optional fields""" + + name: Optional[str] = None + age: Optional[int] = None + gender: Optional[str] = None + work_experience: Optional[int] = None + canada_workex: Optional[int] = None + dep_num: Optional[int] = None + canada_born: Optional[bool] = None + citizen_status: Optional[str] = None + level_of_schooling: Optional[str] = None + fluent_english: Optional[bool] = None + reading_english_scale: Optional[int] = None + speaking_english_scale: Optional[int] = None + writing_english_scale: Optional[int] = None + numeracy_scale: Optional[int] = None + computer_scale: Optional[int] = None + transportation_bool: Optional[bool] = None + caregiver_bool: Optional[bool] = None + housing: Optional[str] = None + income_source: Optional[str] = None + felony_bool: Optional[bool] = None + attending_school: Optional[bool] = None + currently_employed: Optional[bool] = None + substance_use: Optional[bool] = None + time_unemployed: Optional[int] = None + need_mental_health_support_bool: Optional[bool] = None + + +class Client(ClientBase): + """Schema for reading client information, with ORM mode enabled""" + + id: int + + class Config: + '''Class for configuring ORM''' + orm_mode = True diff --git a/app/clients/service/load_data.py b/app/clients/service/load_data.py new file mode 100644 index 00000000..4cfe0514 --- /dev/null +++ b/app/clients/service/load_data.py @@ -0,0 +1,8 @@ +'''Module containing functions for load_data''' + +import pandas as pd +from app.database import engine + + +df = pd.read_csv('data_commontool.csv') +df.to_sql('persons', engine, if_exists='append', index=False) diff --git a/app/clients/service/logic.py b/app/clients/service/logic.py index 0fd826a5..d9f314e3 100644 --- a/app/clients/service/logic.py +++ b/app/clients/service/logic.py @@ -1,10 +1,10 @@ -from typing import List -import pandas as pd -import json -import numpy as np +'''Module containing functions for logic''' + +import os import pickle -from itertools import combinations_with_replacement from itertools import product +import numpy as np + column_intervention = [ 'Life Stabilization', @@ -13,25 +13,23 @@ 'Specialized Services', 'Employment-Related Financial Supports for Job Seekers and Employers', 'Employer Financial Supports', - 'Enhanced Referrals for Skills Development' -] - -#loads the model into logic - -import os + 'Enhanced Referrals for Skills Development'] +# Loads the model into logic current_dir = os.path.dirname(os.path.abspath(__file__)) filename = os.path.join(current_dir, 'model.pkl') +# pylint: disable=consider-using-with model = pickle.load(open(filename, "rb")) def clean_input_data(data): - #translate input into wahtever we trained the model on, numerical data in a specific order - columns = ["age","gender","work_experience","canada_workex","dep_num", "canada_born", - "citizen_status", "level_of_schooling", "fluent_english", "reading_english_scale", - "speaking_english_scale", "writing_english_scale", "numeracy_scale", "computer_scale", - "transportation_bool", "caregiver_bool", "housing", "income_source", "felony_bool", "attending_school", - "currently_employed", "substance_use", "time_unemployed", "need_mental_health_support_bool"] + '''Translate input into what we trained the model on, numerical data in a specific order''' + columns = ["age", "gender", "work_experience", "canada_workex", "dep_num", "canada_born", + "citizen_status","level_of_schooling","fluent_english","reading_english_scale", + "speaking_english_scale", "writing_english_scale","numeracy_scale", + "computer_scale", "transportation_bool", "caregiver_bool", "housing", + "income_source", "felony_bool", "attending_school", "currently_employed", + "substance_use", "time_unemployed", "need_mental_health_support_bool"] demographics = { 'age': data['age'], 'gender': data['gender'], @@ -60,7 +58,8 @@ def clean_input_data(data): } output = [] for column in columns: - data = demographics.get(column, None) #default is None, and if you want to pass a value, can return any value + # If you want to pass a value, can return any value. + data = demographics.get(column, None) if isinstance(data, str): data = convert_text(column, data) output.append(data) @@ -68,8 +67,8 @@ def clean_input_data(data): def convert_text(column, data:str): - # Convert text answers from front end into digits - # TODO: ensure that categorical columns match the valid answers in FormNew.jsx (L131) + '''Convert text answers from front end into digits. + TODO: ensure that categorical columns match the valid answers in FormNew.jsx (L131)''' categorical_cols_integers = [ { "": 0, @@ -132,20 +131,25 @@ def convert_text(column, data:str): return data -#creates 128 possible combinations in order to run every possibility through model + def create_matrix(row): - data = [row.copy() for _ in range(128)] + '''Create 128 possible combos to run every possibility through model''' + data = [row.copy() for _ in range(128)] perms = intervention_permutations(7) data = np.array(data) perms = np.array(perms) - matrix = np.concatenate((data,perms), axis = 1) + matrix = np.concatenate((data,perms), axis = 1) return np.array(matrix) -#create matrix of permutations of 1 and 0 of num length + + def intervention_permutations(num): + '''Create matrix of permutations of 1 and 0 of num length''' perms = list(product([0,1],repeat=num)) return np.array(perms) + def get_baseline_row(row): + '''Get baseline row''' print(type(row)) base_interventions = np.array([0]*7) # no interventions row = np.array(row) @@ -155,37 +159,41 @@ def get_baseline_row(row): return line def intervention_row_to_names(row): + '''Reture names with the intervention''' names = [] for i, value in enumerate(row): - if value == 1: + if value == 1: names.append(column_intervention[i]) return names def process_results(baseline, results): - ##Example: - """ + '''Example: { baseline_probability: 80 #baseline percentage point with no interventions results: [ - (85, [A,B,C]) #new percentange with intervention combinations and list of intervention names + #new percentange with intervention combinations and list of intervention names + (85, [A,B,C]) (89, [B,C]) (91, [D,E]) ] } - """ + ''' result_list= [] for row in results: - percent = row[-1] + percent = row[-1] names = intervention_row_to_names(row) result_list.append((percent,names)) output = { - "baseline": baseline[-1], #if it's an array, want the value inside of the array + #if it's an array, want the value inside of the array + "baseline": baseline[-1], "interventions": result_list, } return output + def interpret_and_calculate(data): + '''Interpret and calculate from data''' raw_data = clean_input_data(data) baseline_row = get_baseline_row(raw_data) baseline_row = baseline_row.reshape(1, -1) @@ -193,25 +201,29 @@ def interpret_and_calculate(data): intervention_rows = create_matrix(raw_data) baseline_prediction = model.predict(baseline_row) intervention_predictions = model.predict(intervention_rows) - intervention_predictions = intervention_predictions.reshape(-1, 1) #want shape to be a vertical column, not a row - result_matrix = np.concatenate((intervention_rows,intervention_predictions), axis = 1) ##CHANGED AXIS - + intervention_predictions = intervention_predictions.reshape(-1, 1) # vertical column + result_matrix = np.concatenate((intervention_rows,intervention_predictions), axis = 1) + # sort this matrix based on prediction # print("RESULT SAMPLE::", result_matrix[:5]) - result_order = result_matrix[:,-1].argsort() #take all rows and only last column, gives back list of indexes sorted - result_matrix = result_matrix[result_order] #indexing the matrix by the order + #take all rows and only last column, gives back list of indexes sorted + #indexing the matrix by the order + result_order = result_matrix[:,-1].argsort() + result_matrix = result_matrix[result_order] # slice matrix to only top N results - result_matrix = result_matrix[-3:,-8:] #-8 for interventions and prediction, want top 3, 3 combinations of intervention + #-8 for interventions and prediction, want top 3, 3 combinations of intervention + result_matrix = result_matrix[-3:,-8:] # post process results if needed ie make list of names for each row results = process_results(baseline_prediction,result_matrix) # build output dict print(f"RESULTS: {results}") return results + if __name__ == "__main__": print("running") - data = { + new_client = { "age": "23", "gender": "1", "work_experience": "1", @@ -237,7 +249,5 @@ def interpret_and_calculate(data): "time_unemployed": "1", "need_mental_health_support_bool": "1" } - # print(data) - results = interpret_and_calculate(data) - print(results) + print(interpret_and_calculate(new_client)) diff --git a/app/clients/service/model.py b/app/clients/service/model.py index 51369ac3..a385ed4b 100644 --- a/app/clients/service/model.py +++ b/app/clients/service/model.py @@ -1,14 +1,17 @@ +'''Creates the model''' +# pylint: disable=all + +import pickle import pandas as pd -import json import numpy as np -import pickle from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestRegressor def prepare_models(): + '''Prepare models''' # Load dataset and define the features and labels - backendCode = pd.read_csv('data_commontool.csv') + backend_code = pd.read_csv('data_commontool.csv') # Define categorical columns and interventions categorical_cols = ['age', 'gender', #bool @@ -21,7 +24,7 @@ def prepare_models(): 'fluent_english', #english level fluency, scale (1-10) 'reading_english_scale', #reading scale (1-10) 'speaking_english_scale', #speaking level comfort (1-10) - 'writing_english_scale', #writing scale (1-10) + 'writing_english_scale', #writing scale (1-10) 'numeracy_scale', #numeracy scale (1-10) 'computer_scale', #computer use scale (1-10) 'transportation_bool', #need transportation support (bool) @@ -45,28 +48,28 @@ def prepare_models(): ] categorical_cols.extend(interventions) # Prepare training data - X_categorical_baseline = backendCode[categorical_cols] + x_categorical_baseline = backendCode[categorical_cols] y_baseline = backendCode['success_rate'] - X_categorical_baseline = np.array(X_categorical_baseline) + x_categorical_baseline = np.array(x_categorical_baseline) y_baseline = np.array(y_baseline) - X_train_baseline, X_test_baseline, y_train_baseline, y_test_baseline = train_test_split( - X_categorical_baseline, y_baseline, test_size=0.2, random_state=42) + x_train_baseline, x_test_baseline, y_train_baseline, y_test_baseline = train_test_split( + x_categorical_baseline, y_baseline, test_size=0.2, random_state=42) rf_model_baseline = RandomForestRegressor(n_estimators=100, random_state=42) - rf_model_baseline.fit(X_train_baseline, y_train_baseline) - - # Example: Predicting on the test set - baseline_predictions = rf_model_baseline.predict(X_test_baseline) + rf_model_baseline.fit(x_train_baseline, y_train_baseline) - return rf_model_baseline + def main(): + '''Prepare and start model''' + print("Start model.") model = prepare_models() pickle.dump(model, open("model.pkl", "wb")) #saves model to the file name input, write binary model = pickle.load(open("model.pkl", "rb")) #read binary + if __name__ == "__main__": main() diff --git a/app/database.py b/app/database.py new file mode 100644 index 00000000..0549527d --- /dev/null +++ b/app/database.py @@ -0,0 +1,31 @@ +'''Module containing functions to define database model''' +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +db_config = { + 'drivername': 'mysql+pymysql', + 'host': 'localhost', + 'port': 3306, + 'username': 'root', + 'password': 'qz325299', + 'database': 'common_assess' +} + +# pylint: disable=line-too-long +engine = create_engine(f"{db_config['drivername']}://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['database']}") + +# Create a base class to define the database model +Base = declarative_base() + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() + +def get_db(): + '''Get database''' + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/app/database_tools.py b/app/database_tools.py new file mode 100644 index 00000000..69fd8219 --- /dev/null +++ b/app/database_tools.py @@ -0,0 +1,42 @@ +'''Module contains functions to load and read database from csv''' +import pandas as pd +from app.database import engine + + +def load_csv_to_db(csv_path, table_name, mode='append'): + """ + mode: + append: Append to the table. + replace: Replace the table. + fail: Raise an error if the table already exists. + """ + df = pd.read_csv(csv_path) + if mode not in ['append', 'replace', 'fail']: + raise ValueError("Mode must be 'append', 'replace', or 'fail'.") + try: + df.to_sql(table_name, engine, if_exists=mode, index=False) + except ValueError as e: + print(f"Error: {e}") + print('success csv to db !') + + +def read_db_to_csv(file_path, table_name, sql=None, mode='w'): + """ + mode: + w: Write to the file. + a: Append to the file. + """ + query = f"SELECT * FROM {table_name}" + if sql is not None: + query = sql + try: + df = pd.read_sql(query, engine) + df.to_csv(file_path, index=False, mode=mode) + except ValueError as e: + print(f"Error: {e}") + print('success db to csv file !') + + +if __name__ == '__main__': + load_csv_to_db(r"C:\Users\zq789\OneDrive\Desktop\common_assess\data_sample.csv", + 'persons', 'replace') diff --git a/app/main.py b/app/main.py index 5b6bf162..db859471 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,16 @@ +'''Module containing main''' + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from app.clients.router import router as clients_router - +from app.database import engine, Base # Still need to create database configuration app = FastAPI() +# Creating database tables if they do not already exist +Base.metadata.create_all(bind=engine) + # Set API endpoints on router app.include_router(clients_router) @@ -16,5 +21,3 @@ allow_methods=["*"], # Allows all methods, including OPTIONS allow_headers=["*"], # Allows all headers ) - - diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..7e679dcc --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Package for unit and integration tests. +""" diff --git a/tests/test.py b/tests/math.py similarity index 86% rename from tests/test.py rename to tests/math.py index a911f0a2..52ef27ae 100644 --- a/tests/test.py +++ b/tests/math.py @@ -1,5 +1,9 @@ -from logic import interpret_and_calculate +'''Module containing some math''' + from itertools import combinations_with_replacement +from itertools import product +# from logic import interpret_and_calculate + # def test_interpret_and_calculate(): # print("running tests") @@ -9,7 +13,6 @@ # result = interpret_and_calculate(data) # print(data) -from itertools import product # Cartesian product of [0, 1] repeated 2 times result = list(product([0, 1], repeat=2)) @@ -20,4 +23,4 @@ result = list(combinations_with_replacement([0, 1], 2)) # Output: [(0, 0), (0, 1), (1, 1)] -print(result) \ No newline at end of file +print(result) diff --git a/tests/test_crud.py b/tests/test_crud.py new file mode 100644 index 00000000..6c99bf1f --- /dev/null +++ b/tests/test_crud.py @@ -0,0 +1,135 @@ +'''Module contains tests for CRUD functions''' +# pylint: disable=unused-import +# pylint: disable=too-few-public-methods + +from unittest.mock import MagicMock +from app.clients.crud import ( + create_client, get_client, get_clients, update_client, delete_client +) +from app.clients.schema import ClientCreate, ClientUpdate, ClientBase + + +class MockClient: + '''Create mock client''' + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def test_create_client(): + '''Test create_client''' + # Mock the database session + mock_db = MagicMock() + + # Provide valid test data matching the ClientCreate schema + client_data = ClientBase( + name="Ann Lu", + age=35, + gender="Male", + work_experience=10, + canada_workex=5, + dep_num=2, + canada_born=True, + citizen_status="Citizen", + level_of_schooling="Bachelor's Degree", + fluent_english=True, + reading_english_scale=5, + speaking_english_scale=5, + writing_english_scale=5, + numeracy_scale=4, + computer_scale=3, + transportation_bool=True, + caregiver_bool=False, + housing="Owned", + income_source="Employment", + felony_bool=False, + attending_school=False, + currently_employed=True, + substance_use=False, + time_unemployed=0, + need_mental_health_support_bool=False + ) + + # Call the function to test + result = create_client(mock_db, client=client_data) + + # Assert that the mock database methods were called correctly + mock_db.add.assert_called_once_with(result) + mock_db.commit.assert_called_once() + + # Verify the result is the same as the input data (or matches expectations) + assert result.name == client_data.name + assert result.age == client_data.age + assert result.gender == client_data.gender + + +def test_get_client(): + '''Test retrieving client by client_id''' + mock_db = MagicMock() + mock_db.query().filter().first.return_value = {"id": 1, "name": "Test Client"} + + result = get_client(mock_db, client_id=1) + assert result == {"id": 1, "name": "Test Client"} + mock_db.query().filter().first.assert_called_once() + + +def test_get_clients(): + '''Test retrieving all clients''' + mock_db = MagicMock() + mock_db.query().offset().limit().all.return_value = [ + {"id": 1, "name": "Client 1"}, + {"id": 2, "name": "Client 2"}, + ] + + result = get_clients(mock_db, skip=0, limit=10) + assert len(result) == 2 + assert result[0]["name"] == "Client 1" + + +def test_update_client(): + '''Test updating a client''' + # Mock the database session + mock_db = MagicMock() + + # Existing client in the database + existing_client = MockClient( + id=1, + name="Old Name", + age=30, + gender="Male", + work_experience=5 + ) + + # Updated client data + updated_data = ClientUpdate( + name="New Name", + age=35, + gender="Male", + work_experience=10 + ) + + # Mock the behavior of query.filter().first() + mock_db.query().filter().first.return_value = existing_client + + # Call the function to test + result = update_client(mock_db, client_id=1, client=updated_data) + + # Assert that the mock database methods were called correctly + mock_db.commit.assert_called_once() + + # Assert that the attributes were updated correctly + assert result.name == updated_data.name + assert result.age == updated_data.age + assert result.gender == updated_data.gender + assert result.work_experience == updated_data.work_experience + + +def test_delete_client(): + '''Test deleting a client''' + mock_db = MagicMock() + mock_db.query().filter().first.return_value = {"id": 1, "name": "Test Client"} + + delete_client(mock_db, client_id=1) + + mock_db.delete.assert_called_once() + mock_db.commit.assert_called_once()