diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a4567f5..5f24de8 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -39,6 +39,10 @@ jobs: run: |- pipenv run python main.py & + - name: Wait for Clima to be ready + run: | + timeout 60 bash -c 'until curl -f http://127.0.0.1:8080; do sleep 1; done' + - name: Test Clima run: |- cd tests diff --git a/pages/lib/charts_sun.py b/pages/lib/charts_sun.py index dd7eea0..d48eb49 100644 --- a/pages/lib/charts_sun.py +++ b/pages/lib/charts_sun.py @@ -16,24 +16,62 @@ from plotly.subplots import make_subplots from pvlib import solarposition from pages.lib.global_variables import Variables, VariableInfo +from pages.lib.utils import separate_filtered_data def monthly_solar(epw_df, si_ip): + # Separate filtered and unfiltered data + # Note: monthly_solar uses two original columns (GLOB_HOR_RAD and DIF_HOR_RAD) + # so we can't use the utility function directly, but we can still use it for separation + filter_info = separate_filtered_data(epw_df) + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + has_filter_marker = filter_info["has_filter_marker"] + + # Get original values if available (for two specific columns) + original_glob_col = f"_{Variables.GLOB_HOR_RAD.col_name}_original" + original_dif_col = f"_{Variables.DIF_HOR_RAD.col_name}_original" + use_original_for_filtered = ( + has_filter_marker + and original_glob_col in epw_df.columns + and original_dif_col in epw_df.columns + ) + + # Calculate monthly averages for unfiltered data g_h_rad_month_ave = ( - epw_df.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ + df_unfiltered.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ Variables.GLOB_HOR_RAD.col_name ] .median() .reset_index() ) dif_h_rad_month_ave = ( - epw_df.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ + df_unfiltered.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ Variables.DIF_HOR_RAD.col_name ] .median() .reset_index() ) + # Calculate monthly averages for filtered data (using original values) + g_h_rad_month_ave_filtered = None + dif_h_rad_month_ave_filtered = None + if df_filtered is not None and len(df_filtered) > 0 and use_original_for_filtered: + g_h_rad_month_ave_filtered = ( + df_filtered.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ + original_glob_col + ] + .median() + .reset_index() + ) + dif_h_rad_month_ave_filtered = ( + df_filtered.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ + original_dif_col + ] + .median() + .reset_index() + ) + # Always show 12 months in horizontal layout fig = make_subplots( rows=1, @@ -42,84 +80,171 @@ def monthly_solar(epw_df, si_ip): shared_yaxes=True, ) + # Track which legend entries have been shown (only show once) + legend_shown = { + "Global": False, + "Diffuse": False, + } + for month_num in range(1, 13): col_idx = month_num - # We only need legend entries for the first pair, since the others repeat. - is_first = col_idx == 1 - fig.add_trace( - go.Scatter( - x=g_h_rad_month_ave.loc[ - g_h_rad_month_ave[Variables.MONTH.col_name] == month_num, - Variables.HOUR.col_name, - ], - y=g_h_rad_month_ave.loc[ - g_h_rad_month_ave[Variables.MONTH.col_name] == month_num, - Variables.GLOB_HOR_RAD.col_name, - ], - fill="tozeroy", - mode="lines", - line_color="orange", - line_width=2, - name="Global", - showlegend=is_first, - customdata=epw_df.loc[ - epw_df[Variables.MONTH.col_name] == month_num, - Variables.MONTH_NAMES.col_name, - ], - hovertemplate=( - "" - + "Global Horizontal Solar Radiation" - + ": %{y:.2f} " - + VariableInfo.from_col_name( - Variables.GLOB_HOR_RAD.col_name - ).get_unit(si_ip) - + "
" - + "Month: %{customdata}
" - + "Hour: %{x}:00
" - + "" # Hides the "secondary box" + # Add filtered data traces (gray) if any + if ( + df_filtered is not None + and len(df_filtered) > 0 + and g_h_rad_month_ave_filtered is not None + ): + month_glob_filtered = g_h_rad_month_ave_filtered.loc[ + g_h_rad_month_ave_filtered[Variables.MONTH.col_name] == month_num + ] + if len(month_glob_filtered) > 0: + # Get the column name from the groupby result (it should be the original column name) + filtered_glob_col = ( + original_glob_col + if original_glob_col in month_glob_filtered.columns + else Variables.GLOB_HOR_RAD.col_name + ) + fig.add_trace( + go.Scatter( + x=month_glob_filtered[Variables.HOUR.col_name], + y=month_glob_filtered[filtered_glob_col], + fill="tozeroy", + mode="lines", + line_color="gray", + line_width=1, + name="Global (Filtered)", + showlegend=False, + customdata=[month_lst[month_num - 1]] * len(month_glob_filtered) + if len(month_glob_filtered) > 0 + else [], + hovertemplate=( + "Filtered Data
" + + "Global Horizontal Solar Radiation" + + ": %{y:.2f} " + + VariableInfo.from_col_name( + Variables.GLOB_HOR_RAD.col_name + ).get_unit(si_ip) + + "
" + + "Month: %{customdata}
" + + "Hour: %{x}:00
" + + "" + ), + ), + row=1, + col=col_idx, + ) + + month_dif_filtered = dif_h_rad_month_ave_filtered.loc[ + dif_h_rad_month_ave_filtered[Variables.MONTH.col_name] == month_num + ] + if len(month_dif_filtered) > 0: + # Get the column name from the groupby result (it should be the original column name) + filtered_dif_col = ( + original_dif_col + if original_dif_col in month_dif_filtered.columns + else Variables.DIF_HOR_RAD.col_name + ) + fig.add_trace( + go.Scatter( + x=month_dif_filtered[Variables.HOUR.col_name], + y=month_dif_filtered[filtered_dif_col], + fill="tozeroy", + mode="lines", + line_color="lightgray", + line_width=1, + name="Diffuse (Filtered)", + showlegend=False, + customdata=[month_lst[month_num - 1]] * len(month_dif_filtered) + if len(month_dif_filtered) > 0 + else [], + hovertemplate=( + "Filtered Data
" + + "Diffuse Horizontal Solar Radiation" + + ": %{y:.2f} " + + VariableInfo.from_col_name( + Variables.DIF_HOR_RAD.col_name + ).get_unit(si_ip) + + "
" + + "Month: %{customdata}
" + + "Hour: %{x}:00
" + + "" + ), + ), + row=1, + col=col_idx, + ) + + # Add unfiltered data traces (normal colors) + month_glob_unfiltered = g_h_rad_month_ave.loc[ + g_h_rad_month_ave[Variables.MONTH.col_name] == month_num + ] + if len(month_glob_unfiltered) > 0: + fig.add_trace( + go.Scatter( + x=month_glob_unfiltered[Variables.HOUR.col_name], + y=month_glob_unfiltered[Variables.GLOB_HOR_RAD.col_name], + fill="tozeroy", + mode="lines", + line_color="orange", + line_width=2, + name="Global", + legendgroup="Global", + showlegend=not legend_shown["Global"], + customdata=[month_lst[month_num - 1]] * len(month_glob_unfiltered), + hovertemplate=( + "" + + "Global Horizontal Solar Radiation" + + ": %{y:.2f} " + + VariableInfo.from_col_name( + Variables.GLOB_HOR_RAD.col_name + ).get_unit(si_ip) + + "
" + + "Month: %{customdata}
" + + "Hour: %{x}:00
" + + "" # Hides the "secondary box" + ), ), - ), - row=1, - col=col_idx, - ) + row=1, + col=col_idx, + ) + if not legend_shown["Global"]: + legend_shown["Global"] = True - fig.add_trace( - go.Scatter( - x=dif_h_rad_month_ave.loc[ - dif_h_rad_month_ave[Variables.MONTH.col_name] == month_num, - Variables.HOUR.col_name, - ], - y=dif_h_rad_month_ave.loc[ - dif_h_rad_month_ave[Variables.MONTH.col_name] == month_num, - Variables.DIF_HOR_RAD.col_name, - ], - fill="tozeroy", - mode="lines", - line_color="dodgerblue", - line_width=2, - name="Diffuse", - showlegend=is_first, - customdata=epw_df.loc[ - epw_df[Variables.MONTH.col_name] == month_num, - Variables.MONTH_NAMES.col_name, - ], - hovertemplate=( - "" - + "Diffuse Horizontal Solar Radiation" - + ": %{y:.2f} " - + VariableInfo.from_col_name( - Variables.DIF_HOR_RAD.col_name - ).get_unit(si_ip) - + "
" - + "Month: %{customdata}
" - + "Hour: %{x}:00
" - + "" # Hides the "secondary box" + month_dif_unfiltered = dif_h_rad_month_ave.loc[ + dif_h_rad_month_ave[Variables.MONTH.col_name] == month_num + ] + if len(month_dif_unfiltered) > 0: + fig.add_trace( + go.Scatter( + x=month_dif_unfiltered[Variables.HOUR.col_name], + y=month_dif_unfiltered[Variables.DIF_HOR_RAD.col_name], + fill="tozeroy", + mode="lines", + line_color="dodgerblue", + line_width=2, + name="Diffuse", + legendgroup="Diffuse", + showlegend=not legend_shown["Diffuse"], + customdata=[month_lst[month_num - 1]] * len(month_dif_unfiltered), + hovertemplate=( + "" + + "Diffuse Horizontal Solar Radiation" + + ": %{y:.2f} " + + VariableInfo.from_col_name( + Variables.DIF_HOR_RAD.col_name + ).get_unit(si_ip) + + "
" + + "Month: %{customdata}
" + + "Hour: %{x}:00
" + + "" # Hides the "secondary box" + ), ), - ), - row=1, - col=col_idx, - ) + row=1, + col=col_idx, + ) + if not legend_shown["Diffuse"]: + legend_shown["Diffuse"] = True fig.update_xaxes(range=[0, 25], row=1, col=col_idx) @@ -140,7 +265,29 @@ def polar_graph(df, meta, global_local, var, si_ip): latitude = float(meta[Variables.LAT.col_name]) longitude = float(meta[Variables.LON.col_name]) time_zone = float(meta[Variables.TIME_ZONE.col_name]) - solpos = df.loc[df[Variables.APPARENT_ELEVATION.col_name] > 0, :] + + # Separate filtered and unfiltered data using utility function + # Note: For "None" variable, pass None to avoid checking for original column + filter_var = None if var == "None" else var + filter_info = separate_filtered_data(df, filter_var) + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + original_var_col = filter_info["original_var_col"] + use_original_for_filtered = filter_info["use_original_for_filtered"] + # Adjust for "None" case + if var == "None": + original_var_col = None + use_original_for_filtered = False + + solpos_unfiltered = df_unfiltered.loc[ + df_unfiltered[Variables.APPARENT_ELEVATION.col_name] > 0, : + ] + solpos_filtered = None + if df_filtered is not None and len(df_filtered) > 0: + solpos_filtered = df_filtered.loc[ + df_filtered[Variables.APPARENT_ELEVATION.col_name] > 0, : + ] + if var != "None": variable = VariableInfo.from_col_name(var) var_unit = variable.get_unit(si_ip) @@ -152,7 +299,10 @@ def polar_graph(df, meta, global_local, var, si_ip): range_z = var_range else: # Set maximum and minimum according to data - data_max, data_min = get_max_min_value(solpos[var]) + if len(solpos_unfiltered) > 0: + data_max, data_min = get_max_min_value(solpos_unfiltered[var]) + else: + data_max, data_min = var_range range_z = [data_min, data_max] tz = "UTC" @@ -161,14 +311,16 @@ def polar_graph(df, meta, global_local, var, si_ip): ) delta = timedelta(days=0, hours=time_zone - 1, minutes=0) times = times - delta - solpos = df.loc[df[Variables.APPARENT_ELEVATION.col_name] > 0, :] if var == "None": var_color = "orange" marker_size = 3 else: - vals = solpos[var] - marker_size = ((vals - vals.min()) / (vals.max() - vals.min()) + 1) * 4 + if len(solpos_unfiltered) > 0: + vals = solpos_unfiltered[var] + marker_size = ((vals - vals.min()) / (vals.max() - vals.min()) + 1) * 4 + else: + marker_size = 3 fig = go.Figure() # draw altitude circles @@ -188,81 +340,180 @@ def polar_graph(df, meta, global_local, var, si_ip): name="", ) ) - # Draw annalemma - if var == "None": - fig.add_trace( - go.Scatterpolar( - r=90 - * np.cos(np.radians(90 - solpos[Variables.APPARENT_ZENITH.col_name])), - theta=solpos[Variables.AZIMUTH.col_name], - mode="markers", - marker_color="orange", - marker_size=marker_size, - marker_line_width=0, - customdata=np.stack( - ( - solpos[Variables.DAY.col_name], - solpos[Variables.MONTH_NAMES.col_name], - solpos[Variables.HOUR.col_name], - solpos[Variables.ELEVATION.col_name], - solpos[Variables.AZIMUTH.col_name], + + # Draw filtered data (gray) if any + if solpos_filtered is not None and len(solpos_filtered) > 0: + if var == "None": + fig.add_trace( + go.Scatterpolar( + r=90 + * np.cos( + np.radians( + 90 - solpos_filtered[Variables.APPARENT_ZENITH.col_name] + ) ), - axis=-1, - ), - hovertemplate="month: %{customdata[1]}" - + "
day: %{customdata[0]:.0f}" - + "
hour: %{customdata[2]:.0f}:00" - + "
sun altitude: %{customdata[3]:.2f}" - + degrees_unit - + "
sun azimuth: %{customdata[4]:.2f}" - + degrees_unit - + "
", - name="", + theta=solpos_filtered[Variables.AZIMUTH.col_name], + mode="markers", + marker_color="gray", + marker_size=marker_size * 0.7, + marker_opacity=0.5, + marker_line_width=0, + customdata=np.stack( + ( + solpos_filtered[Variables.DAY.col_name], + solpos_filtered[Variables.MONTH_NAMES.col_name], + solpos_filtered[Variables.HOUR.col_name], + solpos_filtered[Variables.ELEVATION.col_name], + solpos_filtered[Variables.AZIMUTH.col_name], + ), + axis=-1, + ), + hovertemplate="Filtered Data
month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
", + name="Filtered", + ) ) - ) - else: - fig.add_trace( - go.Scatterpolar( - r=90 - * np.cos(np.radians(90 - solpos[Variables.APPARENT_ZENITH.col_name])), - theta=solpos[Variables.AZIMUTH.col_name], - mode="markers", - marker=dict( - color=solpos[var], - size=marker_size, - line_width=0, - colorscale=var_color, - cmin=range_z[0], - cmax=range_z[1], - colorbar=dict(thickness=30, title=var_unit + "
"), - ), - customdata=np.stack( - ( - solpos[Variables.DAY.col_name], - solpos[Variables.MONTH_NAMES.col_name], - solpos[Variables.HOUR.col_name], - solpos[Variables.ELEVATION.col_name], - solpos[Variables.AZIMUTH.col_name], - solpos[var], + else: + # For filtered data with variable, use original values + if use_original_for_filtered: + filtered_var_vals = solpos_filtered[original_var_col] + else: + filtered_var_vals = solpos_filtered[var] + + fig.add_trace( + go.Scatterpolar( + r=90 + * np.cos( + np.radians( + 90 - solpos_filtered[Variables.APPARENT_ZENITH.col_name] + ) ), - axis=-1, - ), - hovertemplate="month: %{customdata[1]}" - + "
day: %{customdata[0]:.0f}" - + "
hour: %{customdata[2]:.0f}:00" - + "
sun altitude: %{customdata[3]:.2f}" - + degrees_unit - + "
sun azimuth: %{customdata[4]:.2f}" - + degrees_unit - + "
" - + "
" - + var_name - + ": %{customdata[5]:.2f}" - + var_unit - + "", - name="", + theta=solpos_filtered[Variables.AZIMUTH.col_name], + mode="markers", + marker=dict( + color="gray", + size=marker_size * 0.7, + opacity=0.5, + line_width=0, + ), + customdata=np.stack( + ( + solpos_filtered[Variables.DAY.col_name], + solpos_filtered[Variables.MONTH_NAMES.col_name], + solpos_filtered[Variables.HOUR.col_name], + solpos_filtered[Variables.ELEVATION.col_name], + solpos_filtered[Variables.AZIMUTH.col_name], + filtered_var_vals, + ), + axis=-1, + ), + hovertemplate="Filtered Data
month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
" + + "
" + + var_name + + ": %{customdata[5]:.2f}" + + var_unit + + "", + name="Filtered", + ) + ) + + # Draw unfiltered data (normal colors) + if len(solpos_unfiltered) > 0: + if var == "None": + fig.add_trace( + go.Scatterpolar( + r=90 + * np.cos( + np.radians( + 90 - solpos_unfiltered[Variables.APPARENT_ZENITH.col_name] + ) + ), + theta=solpos_unfiltered[Variables.AZIMUTH.col_name], + mode="markers", + marker_color="orange", + marker_size=marker_size, + marker_line_width=0, + customdata=np.stack( + ( + solpos_unfiltered[Variables.DAY.col_name], + solpos_unfiltered[Variables.MONTH_NAMES.col_name], + solpos_unfiltered[Variables.HOUR.col_name], + solpos_unfiltered[Variables.ELEVATION.col_name], + solpos_unfiltered[Variables.AZIMUTH.col_name], + ), + axis=-1, + ), + hovertemplate="month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
", + name="", + ) + ) + else: + fig.add_trace( + go.Scatterpolar( + r=90 + * np.cos( + np.radians( + 90 - solpos_unfiltered[Variables.APPARENT_ZENITH.col_name] + ) + ), + theta=solpos_unfiltered[Variables.AZIMUTH.col_name], + mode="markers", + marker=dict( + color=solpos_unfiltered[var], + size=marker_size, + line_width=0, + colorscale=var_color, + cmin=range_z[0], + cmax=range_z[1], + colorbar=dict(thickness=30, title=var_unit + "
"), + ), + customdata=np.stack( + ( + solpos_unfiltered[Variables.DAY.col_name], + solpos_unfiltered[Variables.MONTH_NAMES.col_name], + solpos_unfiltered[Variables.HOUR.col_name], + solpos_unfiltered[Variables.ELEVATION.col_name], + solpos_unfiltered[Variables.AZIMUTH.col_name], + solpos_unfiltered[var], + ), + axis=-1, + ), + hovertemplate="month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
" + + "
" + + var_name + + ": %{customdata[5]:.2f}" + + var_unit + + "", + name="", + ) ) - ) # draw equinox and sostices for date in pd.to_datetime(["2019-03-21", "2019-06-21", "2019-12-21"]): @@ -355,6 +606,20 @@ def custom_cartesian_solar(df, meta, global_local, var, si_ip): longitude = float(meta[Variables.LON.col_name]) time_zone = float(meta[Variables.TIME_ZONE.col_name]) tz = "UTC" + + # Separate filtered and unfiltered data using utility function + # Note: For "None" variable, pass None to avoid checking for original column + filter_var = None if var == "None" else var + filter_info = separate_filtered_data(df, filter_var) + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + original_var_col = filter_info["original_var_col"] + use_original_for_filtered = filter_info["use_original_for_filtered"] + # Adjust for "None" case + if var == "None": + original_var_col = None + use_original_for_filtered = False + variable = VariableInfo.from_col_name(var) if var != "None": var_unit = variable.get_unit(si_ip) @@ -366,91 +631,177 @@ def custom_cartesian_solar(df, meta, global_local, var, si_ip): range_z = var_range else: # Set maximum and minimum according to data - data_max, data_min = get_max_min_value(df[var]) + if len(df_unfiltered) > 0: + data_max, data_min = get_max_min_value(df_unfiltered[var]) + else: + data_max, data_min = var_range range_z = [data_min, data_max] if var == "None": var_color = "orange" marker_size = 3 else: - vals = df[var] - marker_size = ((vals - vals.min()) / (vals.max() - vals.min()) + 1) * 4 + if len(df_unfiltered) > 0: + vals = df_unfiltered[var] + marker_size = ((vals - vals.min()) / (vals.max() - vals.min()) + 1) * 4 + else: + marker_size = 3 fig = go.Figure() - # draw annalemma - if var == "None": - fig.add_trace( - go.Scatter( - y=df[Variables.ELEVATION.col_name], - x=df[Variables.AZIMUTH.col_name], - mode="markers", - marker_color="orange", - marker_size=marker_size, - marker_line_width=0, - customdata=np.stack( - ( - df[Variables.DAY.col_name], - df[Variables.MONTH_NAMES.col_name], - df[Variables.HOUR.col_name], - df[Variables.ELEVATION.col_name], - df[Variables.AZIMUTH.col_name], + # Draw filtered data (gray) if any + if df_filtered is not None and len(df_filtered) > 0: + if var == "None": + fig.add_trace( + go.Scatter( + y=df_filtered[Variables.ELEVATION.col_name], + x=df_filtered[Variables.AZIMUTH.col_name], + mode="markers", + marker_color="gray", + marker_size=marker_size * 0.7, + marker_opacity=0.5, + marker_line_width=0, + customdata=np.stack( + ( + df_filtered[Variables.DAY.col_name], + df_filtered[Variables.MONTH_NAMES.col_name], + df_filtered[Variables.HOUR.col_name], + df_filtered[Variables.ELEVATION.col_name], + df_filtered[Variables.AZIMUTH.col_name], + ), + axis=-1, ), - axis=-1, - ), - hovertemplate="month: %{customdata[1]}" - + "
day: %{customdata[0]:.0f}" - + "
hour: %{customdata[2]:.0f}:00" - + "
sun altitude: %{customdata[3]:.2f}" - + degrees_unit - + "
sun azimuth: %{customdata[4]:.2f}" - + degrees_unit - + "
", - name="", + hovertemplate="Filtered Data
month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
", + name="Filtered", + ) ) - ) - else: - fig.add_trace( - go.Scatter( - y=df[Variables.ELEVATION.col_name], - x=df[Variables.AZIMUTH.col_name], - mode="markers", - marker=dict( - color=df[var], - size=marker_size, - line_width=0, - colorscale=var_color, - cmin=range_z[0], - cmax=range_z[1], - colorbar=dict(thickness=30, title=var_unit + "
"), - ), - customdata=np.stack( - ( - df[Variables.DAY.col_name], - df[Variables.MONTH_NAMES.col_name], - df[Variables.HOUR.col_name], - df[Variables.ELEVATION.col_name], - df[Variables.AZIMUTH.col_name], - df[var], + else: + # For filtered data with variable, use original values + if use_original_for_filtered: + filtered_var_vals = df_filtered[original_var_col] + else: + filtered_var_vals = df_filtered[var] + + fig.add_trace( + go.Scatter( + y=df_filtered[Variables.ELEVATION.col_name], + x=df_filtered[Variables.AZIMUTH.col_name], + mode="markers", + marker=dict( + color="gray", + size=marker_size * 0.7, + opacity=0.5, + line_width=0, ), - axis=-1, - ), - hovertemplate="month: %{customdata[1]}" - + "
day: %{customdata[0]:.0f}" - + "
hour: %{customdata[2]:.0f}:00" - + "
sun altitude: %{customdata[3]:.2f}" - + degrees_unit - + "
sun azimuth: %{customdata[4]:.2f}" - + degrees_unit - + "
" - + "
" - + var_name - + ": %{customdata[5]:.2f}" - + var_unit - + "", - name="", + customdata=np.stack( + ( + df_filtered[Variables.DAY.col_name], + df_filtered[Variables.MONTH_NAMES.col_name], + df_filtered[Variables.HOUR.col_name], + df_filtered[Variables.ELEVATION.col_name], + df_filtered[Variables.AZIMUTH.col_name], + filtered_var_vals, + ), + axis=-1, + ), + hovertemplate="Filtered Data
month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
" + + "
" + + var_name + + ": %{customdata[5]:.2f}" + + var_unit + + "", + name="Filtered", + ) + ) + + # Draw unfiltered data (normal colors) + if len(df_unfiltered) > 0: + if var == "None": + fig.add_trace( + go.Scatter( + y=df_unfiltered[Variables.ELEVATION.col_name], + x=df_unfiltered[Variables.AZIMUTH.col_name], + mode="markers", + marker_color="orange", + marker_size=marker_size, + marker_line_width=0, + customdata=np.stack( + ( + df_unfiltered[Variables.DAY.col_name], + df_unfiltered[Variables.MONTH_NAMES.col_name], + df_unfiltered[Variables.HOUR.col_name], + df_unfiltered[Variables.ELEVATION.col_name], + df_unfiltered[Variables.AZIMUTH.col_name], + ), + axis=-1, + ), + hovertemplate="month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
", + name="", + ) + ) + else: + fig.add_trace( + go.Scatter( + y=df_unfiltered[Variables.ELEVATION.col_name], + x=df_unfiltered[Variables.AZIMUTH.col_name], + mode="markers", + marker=dict( + color=df_unfiltered[var], + size=marker_size, + line_width=0, + colorscale=var_color, + cmin=range_z[0], + cmax=range_z[1], + colorbar=dict(thickness=30, title=var_unit + "
"), + ), + customdata=np.stack( + ( + df_unfiltered[Variables.DAY.col_name], + df_unfiltered[Variables.MONTH_NAMES.col_name], + df_unfiltered[Variables.HOUR.col_name], + df_unfiltered[Variables.ELEVATION.col_name], + df_unfiltered[Variables.AZIMUTH.col_name], + df_unfiltered[var], + ), + axis=-1, + ), + hovertemplate="month: %{customdata[1]}" + + "
day: %{customdata[0]:.0f}" + + "
hour: %{customdata[2]:.0f}:00" + + "
sun altitude: %{customdata[3]:.2f}" + + degrees_unit + + "
sun azimuth: %{customdata[4]:.2f}" + + degrees_unit + + "
" + + "
" + + var_name + + ": %{customdata[5]:.2f}" + + var_unit + + "", + name="", + ) ) - ) # draw equinox and sostices for date in pd.to_datetime(["2019-03-21", "2019-06-21", "2019-12-21"]): diff --git a/pages/lib/template_graphs.py b/pages/lib/template_graphs.py index d76665e..797249b 100644 --- a/pages/lib/template_graphs.py +++ b/pages/lib/template_graphs.py @@ -4,21 +4,29 @@ from plotly.subplots import make_subplots from config import UnitSystem -from pages.lib.utils import get_max_min_value +from pages.lib.utils import ( + get_max_min_value, + has_filtered_data, + get_variable_info, + get_variable_range, + get_original_column_values, + calculate_daily_statistics, + unpack_variable_info, +) import dash_bootstrap_components as dbc from .global_scheme import month_lst, template, tight_margins, WIND_ROSE_BINS from pages.lib.global_variables import Variables, VariableInfo -from .utils import code_timer, determine_month_and_hour_filter +from .utils import code_timer, determine_month_and_hour_filter, separate_filtered_data def violin(df, var, global_local, si_ip): """Return day night violin based on the 'var' col""" mask_day = (df[Variables.HOUR.col_name] >= 8) & (df[Variables.HOUR.col_name] < 20) mask_night = (df[Variables.HOUR.col_name] < 8) | (df[Variables.HOUR.col_name] >= 20) - variable = VariableInfo.from_col_name(var) - var_unit = variable.get_unit(si_ip) - var_range = variable.get_range(si_ip) - var_name = variable.get_name() + var_info = get_variable_info(var, si_ip) + var_unit, var_range, var_name = unpack_variable_info( + var_info, ["var_unit", "var_range", "var_name"] + ) data_day = df.loc[mask_day, var] data_night = df.loc[mask_night, var] @@ -84,99 +92,287 @@ def violin(df, var, global_local, si_ip): @code_timer def yearly_profile(df, var, global_local, si_ip): """Return yearly profile figure based on the 'var' col.""" - variable = VariableInfo.from_col_name(var) - var_unit = variable.get_unit(si_ip) - var_range = variable.get_range(si_ip) - var_name = variable.get_name() - var_color = variable.get_color() + var_info = get_variable_info(var, si_ip) + var_unit, var_range, var_name, var_color = unpack_variable_info(var_info) + + # Separate filtered and unfiltered data using utility function + filter_info = separate_filtered_data(df, var) + has_filter_marker = filter_info["has_filter_marker"] + filtered_mask = filter_info["filtered_mask"] + original_var_col = filter_info["original_var_col"] + use_original_for_filtered = filter_info["use_original_for_filtered"] + # Calculate y-axis range - use original values if available to keep range consistent if global_local == "global": # Set Global values for Max and minimum range_y = var_range else: # Set maximum and minimum according to data - data_max, data_min = get_max_min_value(df[var]) - range_y = [data_min, data_max] + # If filtering is active, use original values to maintain consistent y-axis range + if ( + has_filter_marker + and use_original_for_filtered + and filtered_mask is not None + and filtered_mask.any() + ): + # Combine unfiltered values and original filtered values for range calculation + values_for_range = pd.concat( + [ + df[~filtered_mask][var], + df[filtered_mask][original_var_col], + ] + ).dropna() + # Use combined values if available, otherwise fallback to current values + range_y = get_variable_range( + var, + df, + "local", + si_ip, + use_original_for_range=len(values_for_range) > 0, + original_values=values_for_range if len(values_for_range) > 0 else None, + ) + else: + range_y = get_variable_range(var, df, "local", si_ip) var_single_color = var_color[len(var_color) // 2] custom_ylim = range_y - # Get min, max, and mean of each day - dbt_day = df.groupby(np.arange(len(df.index)) // 24)[var].agg( - ["min", "max", "mean"] - ) - trace1 = go.Bar( - x=df[Variables.UTC_TIME.col_name].dt.date.unique(), - y=dbt_day["max"] - dbt_day["min"], - base=dbt_day["min"], - marker_color=var_single_color, - marker_opacity=0.3, - name=var_name + " Range", - customdata=np.stack( - ( - dbt_day["mean"], - df.iloc[::24, :][Variables.MONTH_NAMES.col_name], - df.iloc[::24, :][Variables.DAY.col_name], + # Get all unique dates from the full dataframe for consistent x-axis alignment + all_dates = sorted(df[Variables.UTC_TIME.col_name].dt.date.unique()) + + # Get min, max, and mean of each day for unfiltered and filtered data + if has_filter_marker and filtered_mask is not None: + # Use already separated data from filter_info + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + + # Calculate statistics for unfiltered data + dbt_day_unfiltered = calculate_daily_statistics(df_unfiltered, var) + + # Calculate statistics for filtered data (using original values) + if has_filtered_data(df_filtered) and use_original_for_filtered: + dbt_day_filtered = calculate_daily_statistics(df_filtered, original_var_col) + else: + dbt_day_filtered = None + else: + # No filtering, use full dataframe + df_unfiltered = df + df_filtered = None + dbt_day_unfiltered = calculate_daily_statistics(df, var) + dbt_day_filtered = None + + traces = [] + + # Add filtered data traces (gray) if any filtered data exists + if ( + has_filter_marker + and filtered_mask is not None + and filtered_mask.any() + and dbt_day_filtered is not None + and len(dbt_day_filtered) > 0 + ): + # Reindex to all_dates to ensure consistent x-axis alignment + dbt_day_filtered_reindexed = dbt_day_filtered.reindex(all_dates) + + # Create a mapping from date to month/day names for customdata + df_filtered_date_map = df_filtered.copy() + df_filtered_date_map["_date"] = df_filtered_date_map[ + Variables.UTC_TIME.col_name + ].dt.date + # Get first occurrence of each date for month/day names + date_to_metadata_filtered = df_filtered_date_map.groupby("_date").first() + + # Build customdata arrays aligned with all_dates + filtered_month_names = [ + date_to_metadata_filtered.loc[date, Variables.MONTH_NAMES.col_name] + if date in date_to_metadata_filtered.index + else "" + for date in all_dates + ] + filtered_day_names = [ + date_to_metadata_filtered.loc[date, Variables.DAY.col_name] + if date in date_to_metadata_filtered.index + else "" + for date in all_dates + ] + + trace1_filtered = go.Bar( + x=all_dates, + y=dbt_day_filtered_reindexed["max"] - dbt_day_filtered_reindexed["min"], + base=dbt_day_filtered_reindexed["min"], + marker_color="gray", + marker_opacity=0.3, + name=var_name + " Range (Filtered)", + customdata=np.stack( + ( + dbt_day_filtered_reindexed["mean"].values, + filtered_month_names, + filtered_day_names, + ), + axis=-1, ), - axis=-1, - ), - hovertemplate=( - "Max: %{y:.2f} " - + var_unit - + "
Min: %{base:.2f} " - + var_unit - + "
Ave : %{customdata[0]:.2f} " - + var_unit - + "
Month: %{customdata[1]}
Day: %{customdata[2]}
" - ), - ) + hovertemplate=( + "Filtered Data
Max: %{y:.2f} " + + var_unit + + "
Min: %{base:.2f} " + + var_unit + + "
Ave : %{customdata[0]:.2f} " + + var_unit + + "
Month: %{customdata[1]}
Day: %{customdata[2]}
" + ), + ) + traces.append(trace1_filtered) + + trace2_filtered = go.Scatter( + x=all_dates, + y=dbt_day_filtered_reindexed["mean"], + name="Average " + var_name + " (Filtered)", + mode="lines", + marker_color="lightgray", + marker_opacity=1, + line=dict(color="lightgray", width=2), + customdata=np.stack( + ( + dbt_day_filtered_reindexed["mean"].values, + filtered_month_names, + filtered_day_names, + ), + axis=-1, + ), + hovertemplate=( + "Filtered Data
Ave : %{customdata[0]:.2f} " + + var_unit + + "
Month: %{customdata[1]}
Day: %{customdata[2]}
" + ), + ) + traces.append(trace2_filtered) + + # Add unfiltered data traces (normal colors) + if len(dbt_day_unfiltered) > 0: + # Reindex to all_dates to ensure consistent x-axis alignment + dbt_day_unfiltered_reindexed = dbt_day_unfiltered.reindex(all_dates) + + # Create a mapping from date to month/day names for customdata + df_unfiltered_date_map = df_unfiltered.copy() + df_unfiltered_date_map["_date"] = df_unfiltered_date_map[ + Variables.UTC_TIME.col_name + ].dt.date + # Get first occurrence of each date for month/day names + date_to_metadata = df_unfiltered_date_map.groupby("_date").first() + + # Build customdata arrays aligned with all_dates + unfiltered_month_names = [ + date_to_metadata.loc[date, Variables.MONTH_NAMES.col_name] + if date in date_to_metadata.index + else "" + for date in all_dates + ] + unfiltered_day_names = [ + date_to_metadata.loc[date, Variables.DAY.col_name] + if date in date_to_metadata.index + else "" + for date in all_dates + ] - trace2 = go.Scatter( - x=df[Variables.UTC_TIME.col_name].dt.date.unique(), - y=dbt_day["mean"], - name="Average " + var_name, - mode="lines", - marker_color=var_single_color, - marker_opacity=1, - customdata=np.stack( - ( - dbt_day["mean"], - df.iloc[::24, :][Variables.MONTH_NAMES.col_name], - df.iloc[::24, :][Variables.DAY.col_name], + trace1 = go.Bar( + x=all_dates, + y=dbt_day_unfiltered_reindexed["max"] - dbt_day_unfiltered_reindexed["min"], + base=dbt_day_unfiltered_reindexed["min"], + marker_color=var_single_color, + marker_opacity=0.3, + name=var_name + " Range", + customdata=np.stack( + ( + dbt_day_unfiltered_reindexed["mean"].values, + unfiltered_month_names, + unfiltered_day_names, + ), + axis=-1, ), - axis=-1, - ), - hovertemplate=( - "Ave : %{customdata[0]:.2f} " - + var_unit - + "
Month: %{customdata[1]}
Day: %{customdata[2]}
" - ), - ) + hovertemplate=( + "Max: %{y:.2f} " + + var_unit + + "
Min: %{base:.2f} " + + var_unit + + "
Ave : %{customdata[0]:.2f} " + + var_unit + + "
Month: %{customdata[1]}
Day: %{customdata[2]}
" + ), + ) + traces.append(trace1) + + trace2 = go.Scatter( + x=all_dates, + y=dbt_day_unfiltered_reindexed["mean"], + name="Average " + var_name, + mode="lines", + marker_color=var_single_color, + marker_opacity=1, + customdata=np.stack( + ( + dbt_day_unfiltered_reindexed["mean"].values, + unfiltered_month_names, + unfiltered_day_names, + ), + axis=-1, + ), + hovertemplate=( + "Ave : %{customdata[0]:.2f} " + + var_unit + + "
Month: %{customdata[1]}
Day: %{customdata[2]}
" + ), + ) + traces.append(trace2) if var == Variables.DBT.col_name: # plot ashrae adaptive comfort limits (80%) - lo80 = ( - df.groupby(Variables.DOY.col_name)[Variables.ADAPTIVE_CMF_80_LOW.col_name] - .mean() - .values - ) - hi80 = ( - df.groupby(Variables.DOY.col_name)[Variables.ADAPTIVE_CMF_80_UP.col_name] - .mean() - .values - ) - rmt = ( - df.groupby(Variables.DOY.col_name)[Variables.ADAPTIVE_CMF_RMT.col_name] - .mean() - .values - ) + # Group by DOY and get mean values + doy_grouped = df.groupby(Variables.DOY.col_name) + lo80_by_doy = doy_grouped[Variables.ADAPTIVE_CMF_80_LOW.col_name].mean() + hi80_by_doy = doy_grouped[Variables.ADAPTIVE_CMF_80_UP.col_name].mean() + rmt_by_doy = doy_grouped[Variables.ADAPTIVE_CMF_RMT.col_name].mean() + + # Map DOY values to dates + df_with_date_doy = df.copy() + df_with_date_doy["_date"] = df_with_date_doy[ + Variables.UTC_TIME.col_name + ].dt.date + date_to_doy = df_with_date_doy.groupby("_date")[Variables.DOY.col_name].first() + + # Align ASHRAE values to all_dates + lo80_aligned = [ + lo80_by_doy.get( + date_to_doy.get(date, 1), + lo80_by_doy.iloc[0] if len(lo80_by_doy) > 0 else 0, + ) + for date in all_dates + ] + hi80_aligned = [ + hi80_by_doy.get( + date_to_doy.get(date, 1), + hi80_by_doy.iloc[0] if len(hi80_by_doy) > 0 else 0, + ) + for date in all_dates + ] + rmt_aligned = [ + rmt_by_doy.get( + date_to_doy.get(date, 1), + rmt_by_doy.iloc[0] if len(rmt_by_doy) > 0 else 0, + ) + for date in all_dates + ] + # set color https://github.com/CenterForTheBuiltEnvironment/clima/issues/113 implementation - var_bar_colors = np.where((rmt > 40) | (rmt < 10), "lightgray", "darkgray") + var_bar_colors = np.where( + (np.array(rmt_aligned) > 40) | (np.array(rmt_aligned) < 10), + "lightgray", + "darkgray", + ) trace3 = go.Bar( - x=df[Variables.UTC_TIME.col_name].dt.date.unique(), - y=hi80 - lo80, - base=lo80, + x=all_dates, + y=np.array(hi80_aligned) - np.array(lo80_aligned), + base=lo80_aligned, name="ASHRAE adaptive comfort (80%)", marker_color=var_bar_colors, marker_opacity=0.5, @@ -186,21 +382,29 @@ def yearly_profile(df, var, global_local, si_ip): ) # plot ashrae adaptive comfort limits (90%) - lo90 = ( - df.groupby(Variables.DOY.col_name)[Variables.ADAPTIVE_CMF_90_LOW.col_name] - .mean() - .values - ) - hi90 = ( - df.groupby(Variables.DOY.col_name)[Variables.ADAPTIVE_CMF_90_UP.col_name] - .mean() - .values - ) + lo90_by_doy = doy_grouped[Variables.ADAPTIVE_CMF_90_LOW.col_name].mean() + hi90_by_doy = doy_grouped[Variables.ADAPTIVE_CMF_90_UP.col_name].mean() + + # Align ASHRAE values to all_dates + lo90_aligned = [ + lo90_by_doy.get( + date_to_doy.get(date, 1), + lo90_by_doy.iloc[0] if len(lo90_by_doy) > 0 else 0, + ) + for date in all_dates + ] + hi90_aligned = [ + hi90_by_doy.get( + date_to_doy.get(date, 1), + hi90_by_doy.iloc[0] if len(hi90_by_doy) > 0 else 0, + ) + for date in all_dates + ] trace4 = go.Bar( - x=df[Variables.UTC_TIME.col_name].dt.date.unique(), - y=hi90 - lo90, - base=lo90, + x=all_dates, + y=np.array(hi90_aligned) - np.array(lo90_aligned), + base=lo90_aligned, name="ASHRAE adaptive comfort (90%)", marker_color=var_bar_colors, marker_opacity=0.5, @@ -208,31 +412,31 @@ def yearly_profile(df, var, global_local, si_ip): "Max: %{y:.2f} " + var_unit + "Min: %{base:.2f} " + var_unit ), ) - data = [trace3, trace4, trace1, trace2] + # Insert ASHRAE traces before the main traces + traces = [trace3, trace4] + traces elif var == Variables.RH.col_name: # plot relative Humidity limits (30-70%) - lo_rh = [30] * 365 - hi_rh = [70] * 365 - lo_rh_df = pd.DataFrame({Variables.LO_RH.col_name: lo_rh}) - hi_rh_df = pd.DataFrame({Variables.HI_RH.col_name: hi_rh}) + # Align to all_dates length + lo_rh = [30] * len(all_dates) + hi_rh = [70] * len(all_dates) trace3 = go.Bar( - x=df[Variables.UTC_TIME.col_name].dt.date.unique(), - y=hi_rh_df[Variables.HI_RH.col_name] - lo_rh_df[Variables.LO_RH.col_name], - base=lo_rh_df[Variables.LO_RH.col_name], + x=all_dates, + y=np.array(hi_rh) - np.array(lo_rh), + base=lo_rh, name="humidity comfort band", marker_opacity=0.3, marker_color="silver", ) - data = [trace3, trace1, trace2] + # Insert humidity comfort band before the main traces + traces = [trace3] + traces - else: - data = [trace1, trace2] + # traces already contains the main traces (trace1, trace2, and filtered versions if any) fig = go.Figure( - data=data, layout=go.Layout(barmode="overlay", bargap=0, margin=tight_margins) + data=traces, layout=go.Layout(barmode="overlay", bargap=0, margin=tight_margins) ) fig.update_xaxes( @@ -265,25 +469,39 @@ def yearly_profile(df, var, global_local, si_ip): # @code_timer def daily_profile(df, var, global_local, si_ip): """Return the daily profile based on the 'var' col.""" - variable = VariableInfo.from_col_name(var) - var_name = variable.get_name() - var_unit = variable.get_unit(si_ip) - var_range = variable.get_range(si_ip) - var_color = variable.get_color() - if global_local == "global": - # Set Global values for Max and minimum - range_y = var_range - else: - # Set maximum and minimum according to data - data_max, data_min = get_max_min_value(df[var]) - range_y = [data_min, data_max] + var_info = get_variable_info(var, si_ip) + var_name, var_unit, var_color = unpack_variable_info( + var_info, ["var_name", "var_unit", "var_color"] + ) + range_y = get_variable_range(var, df, global_local, si_ip) var_single_color = var_color[len(var_color) // 2] + + # Separate filtered and unfiltered data using utility function + filter_info = separate_filtered_data(df, var) + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + original_var_col = filter_info["original_var_col"] + use_original_for_filtered = filter_info["use_original_for_filtered"] + + # Calculate monthly averages for unfiltered data var_month_ave = ( - df.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[var] + df_unfiltered.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[var] .median() .reset_index() ) + + # Calculate monthly averages for filtered data (using original values) + var_month_ave_filtered = None + if has_filtered_data(df_filtered) and use_original_for_filtered: + var_month_ave_filtered = ( + df_filtered.groupby([Variables.MONTH.col_name, Variables.HOUR.col_name])[ + original_var_col + ] + .median() + .reset_index() + ) + fig = make_subplots( rows=1, cols=12, @@ -292,55 +510,122 @@ def daily_profile(df, var, global_local, si_ip): ) for i in range(12): - fig.add_trace( - go.Scatter( - x=df.loc[ - df[Variables.MONTH.col_name] == i + 1, Variables.HOUR.col_name - ], - y=df.loc[df[Variables.MONTH.col_name] == i + 1, var], - mode="markers", - marker_color=var_single_color, - opacity=0.5, - marker_size=3, - name=month_lst[i], - showlegend=False, - customdata=df.loc[ - df[Variables.MONTH.col_name] == i + 1, - Variables.MONTH_NAMES.col_name, - ], - hovertemplate=( - "" - + var - + ": %{y:.2f} " - + var_unit - + "
Month: %{customdata}
Hour: %{x}:00
" + month_data_unfiltered = df_unfiltered.loc[ + df_unfiltered[Variables.MONTH.col_name] == i + 1 + ] + month_data_filtered = None + if has_filtered_data(df_filtered): + month_data_filtered = df_filtered.loc[ + df_filtered[Variables.MONTH.col_name] == i + 1 + ] + + # Add filtered data scatter (gray) if any + if month_data_filtered is not None and len(month_data_filtered) > 0: + filtered_var_values = ( + month_data_filtered[original_var_col] + if use_original_for_filtered + else month_data_filtered[var] + ) + fig.add_trace( + go.Scatter( + x=month_data_filtered[Variables.HOUR.col_name], + y=filtered_var_values, + mode="markers", + marker_color="gray", + opacity=0.3, + marker_size=2, + name=month_lst[i] + " (Filtered)", + showlegend=False, + customdata=month_data_filtered[Variables.MONTH_NAMES.col_name], + hovertemplate=( + "Filtered Data
" + + var + + ": %{y:.2f} " + + var_unit + + "
Month: %{customdata}
Hour: %{x}:00
" + ), ), - ), - row=1, - col=i + 1, - ) + row=1, + col=i + 1, + ) - fig.add_trace( - go.Scatter( - x=var_month_ave.loc[ - var_month_ave[Variables.MONTH.col_name] == i + 1, - Variables.HOUR.col_name, - ], - y=var_month_ave.loc[ - var_month_ave[Variables.MONTH.col_name] == i + 1, var - ], - mode="lines", - line_color=var_single_color, - line_width=3, - name=None, - showlegend=False, - hovertemplate=( - "" + var + ": %{y:.2f} " + var_unit + "
Hour: %{x}:00
" + # Add filtered data median line (lightgray) if available + if var_month_ave_filtered is not None and len(var_month_ave_filtered) > 0: + month_ave_filtered = var_month_ave_filtered.loc[ + var_month_ave_filtered[Variables.MONTH.col_name] == i + 1 + ] + if len(month_ave_filtered) > 0: + fig.add_trace( + go.Scatter( + x=month_ave_filtered[Variables.HOUR.col_name], + y=month_ave_filtered[original_var_col], + mode="lines", + line_color="lightgray", + line_width=2, + name=None, + showlegend=False, + hovertemplate=( + "Filtered Data
" + + var + + ": %{y:.2f} " + + var_unit + + "
Hour: %{x}:00
" + ), + ), + row=1, + col=i + 1, + ) + + # Add unfiltered data scatter (normal color) + if len(month_data_unfiltered) > 0: + fig.add_trace( + go.Scatter( + x=month_data_unfiltered[Variables.HOUR.col_name], + y=month_data_unfiltered[var], + mode="markers", + marker_color=var_single_color, + opacity=0.5, + marker_size=3, + name=month_lst[i], + showlegend=False, + customdata=month_data_unfiltered[Variables.MONTH_NAMES.col_name], + hovertemplate=( + "" + + var + + ": %{y:.2f} " + + var_unit + + "
Month: %{customdata}
Hour: %{x}:00
" + ), ), - ), - row=1, - col=i + 1, - ) + row=1, + col=i + 1, + ) + + # Add unfiltered data median line (normal color) + month_ave_unfiltered = var_month_ave.loc[ + var_month_ave[Variables.MONTH.col_name] == i + 1 + ] + if len(month_ave_unfiltered) > 0: + fig.add_trace( + go.Scatter( + x=month_ave_unfiltered[Variables.HOUR.col_name], + y=month_ave_unfiltered[var], + mode="lines", + line_color=var_single_color, + line_width=3, + name=None, + showlegend=False, + hovertemplate=( + "" + + var + + ": %{y:.2f} " + + var_unit + + "
Hour: %{x}:00
" + ), + ), + row=1, + col=i + 1, + ) fig.update_xaxes(range=[0, 25], row=1, col=i + 1) fig.update_yaxes(range=range_y, row=1, col=i + 1) @@ -373,10 +658,10 @@ def heatmap_with_filter( z_range=None, ): """General function that returns a heatmap.""" - variable = VariableInfo.from_col_name(var) - var_unit = variable.get_unit(si_ip) - var_range = variable.get_range(si_ip) - var_color = variable.get_color() + var_info = get_variable_info(var, si_ip) + var_unit, var_range, var_color = unpack_variable_info( + var_info, ["var_unit", "var_range", "var_color"] + ) has_global_filter_marker = "_is_filtered" in df.columns global_filter_mask = None @@ -407,13 +692,10 @@ def heatmap_with_filter( # For category variables (e.g., UTCI categories), always use global range # to ensure consistent color mapping regardless of data range - if "_categories" in var or global_local == "global": - # Set Global values for Max and minimum + if "_categories" in var: range_z = var_range else: - # Set maximum and minimum according to data - data_max, data_min = get_max_min_value(df[var]) - range_z = [data_min, data_max] + range_z = get_variable_range(var, df, global_local, si_ip) fig = go.Figure() has_filter_marker = "_is_filtered" in df.columns @@ -421,11 +703,7 @@ def heatmap_with_filter( if has_filter_marker and df["_is_filtered"].any(): filtered_mask = df["_is_filtered"] if filtered_mask.any(): - original_col = f"_{var}_original" - if original_col in df.columns: - filtered_values = df[original_col].copy() - else: - filtered_values = df[var].copy() + filtered_values = get_original_column_values(df, var) filtered_values[~filtered_mask] = None @@ -543,18 +821,11 @@ def heatmap_with_filter( def heatmap(df, var, global_local, si_ip): """General function that returns a heatmap.""" - variable = VariableInfo.from_col_name(var) - var_unit = variable.get_unit(si_ip) - var_range = variable.get_range(si_ip) - var_color = variable.get_color() - - if global_local == "global": - # Set Global values for Max and minimum - range_z = var_range - else: - # Set maximum and minimum according to data - data_max, data_min = get_max_min_value(df[var]) - range_z = [data_min, data_max] + var_info = get_variable_info(var, si_ip) + var_unit, var_range, var_color = unpack_variable_info( + var_info, ["var_unit", "var_range", "var_color"] + ) + range_z = get_variable_range(var, df, global_local, si_ip) fig = go.Figure() has_filter_marker = "_is_filtered" in df.columns @@ -562,11 +833,7 @@ def heatmap(df, var, global_local, si_ip): if has_filter_marker and df["_is_filtered"].any(): filtered_mask = df["_is_filtered"] if filtered_mask.any(): - original_col = f"_{var}_original" - if original_col in df.columns: - filtered_values = df[original_col].copy() - else: - filtered_values = df[var].copy() + filtered_values = get_original_column_values(df, var) filtered_values[~filtered_mask] = None @@ -856,9 +1123,21 @@ def thermal_stress_stacked_barchart( "#A3302B", "#6B1F18", ] + + # Check if there's a filter marker before applying time filter + has_filter_marker = "_is_filtered" in df.columns + global_filter_mask = None + if has_filter_marker: + global_filter_mask = df["_is_filtered"].copy() + df = filter_df_by_month_and_hour( df, time_filter, month, hour, invert_month, invert_hour, var ) + + # Restore filter marker after time filtering + if has_filter_marker and global_filter_mask is not None: + df["_is_filtered"] = global_filter_mask + start_month, end_month, start_hour, end_hour = determine_month_and_hour_filter( month, hour, invert_month, invert_hour ) @@ -873,33 +1152,91 @@ def thermal_stress_stacked_barchart( style={"text-align": "center", "marginTop": "2rem"}, ), ) + + # Separate filtered and unfiltered data using utility function + filter_info = separate_filtered_data(df, var) + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + has_filtered_data_flag = has_filtered_data(df_filtered) + isNormalized = True if normalize else False + + # Calculate data for unfiltered if isNormalized: - new_df = ( - df.groupby(Variables.MONTH.col_name)[var] + new_df_unfiltered = ( + df_unfiltered.groupby(Variables.MONTH.col_name)[var] .value_counts(normalize=True) .unstack(var) .fillna(0) ) - new_df = new_df.set_axis(categories, axis=1) - new_df.reset_index(inplace=True) + new_df_unfiltered = new_df_unfiltered.set_axis(categories, axis=1) + new_df_unfiltered.reset_index(inplace=True) else: - new_df = ( - df.groupby(Variables.MONTH.col_name)[var] + new_df_unfiltered = ( + df_unfiltered.groupby(Variables.MONTH.col_name)[var] .value_counts() .unstack(var) .fillna(0) ) - new_df = new_df.set_axis(categories, axis=1) - new_df.reset_index(inplace=True) + new_df_unfiltered = new_df_unfiltered.set_axis(categories, axis=1) + new_df_unfiltered.reset_index(inplace=True) + + # Calculate data for filtered (if any) + new_df_filtered = None + if has_filtered_data_flag: + # Use original values for filtered data if available + original_var_col = f"_{var}_original" + use_original = original_var_col in df_filtered.columns + + if use_original: + # Create a temporary column with original values for calculation + df_filtered_temp = df_filtered.copy() + df_filtered_temp[var] = df_filtered_temp[original_var_col] + else: + df_filtered_temp = df_filtered + + if isNormalized: + new_df_filtered = ( + df_filtered_temp.groupby(Variables.MONTH.col_name)[var] + .value_counts(normalize=True) + .unstack(var) + .fillna(0) + ) + new_df_filtered = new_df_filtered.set_axis(categories, axis=1) + new_df_filtered.reset_index(inplace=True) + else: + new_df_filtered = ( + df_filtered_temp.groupby(Variables.MONTH.col_name)[var] + .value_counts() + .unstack(var) + .fillna(0) + ) + new_df_filtered = new_df_filtered.set_axis(categories, axis=1) + new_df_filtered.reset_index(inplace=True) go.Figure() data = [] + + # Filtered data traces removed - no gray filtering effect for thermal stress chart + + # Add unfiltered data traces (normal colors) for i in range(len(categories)): x_data = list(range(0, 12)) - y_data = [ - catch(lambda: new_df.iloc[mth][categories[i]]) for mth in range(0, 12) - ] + y_data = [] + for mth in range(0, 12): + month_idx = mth + 1 # month index (1-12) + # Check if this month exists in unfiltered data + month_rows = new_df_unfiltered[ + new_df_unfiltered[Variables.MONTH.col_name] == month_idx + ] + if len(month_rows) > 0: + try: + val = month_rows.iloc[0][categories[i]] + y_data.append(val if not pd.isna(val) else 0) + except (KeyError, IndexError, TypeError): + y_data.append(0) + else: + y_data.append(0) data.append( go.Bar( x=x_data, @@ -943,8 +1280,11 @@ def thermal_stress_stacked_barchart( linecolor="black", mirror=True, ) - # Get available months from filtered data - available_months = sorted(new_df[Variables.MONTH.col_name].unique()) + # Get available months from unfiltered data (or combined if no filter) + available_months = sorted(new_df_unfiltered[Variables.MONTH.col_name].unique()) + if has_filtered_data_flag and new_df_filtered is not None: + filtered_months = sorted(new_df_filtered[Variables.MONTH.col_name].unique()) + available_months = sorted(set(available_months + filtered_months)) fig.update_xaxes( dict( @@ -977,19 +1317,32 @@ def barchart(df, var, time_filter_info, data_filter_info, normalize, si_ip): filter_name = filter_variable.get_name() filter_unit = filter_variable.get_unit(si_ip) - var_variable = VariableInfo.from_col_name(var) - var_unit = var_variable.get_unit(si_ip) - var_name = var_variable.get_name() - var_color = var_variable.get_color() + var_info = get_variable_info(var, si_ip) + var_unit, var_name, var_color = unpack_variable_info( + var_info, ["var_unit", "var_name", "var_color"] + ) color_below = var_color[0] color_above = var_color[-1] color_in = var_color[len(var_color) // 2] new_df = df.copy() + + # Separate filtered and unfiltered data using utility function + filter_info = separate_filtered_data(new_df, var) + has_filter_marker = filter_info["has_filter_marker"] + filtered_mask = filter_info["filtered_mask"] + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + original_var_col = filter_info["original_var_col"] + use_original_for_filtered = filter_info["use_original_for_filtered"] + month_in = [] month_below = [] month_above = [] + month_in_filtered = [] + month_below_filtered = [] + month_above_filtered = [] min_val = str(min_val) max_val = str(max_val) @@ -1002,25 +1355,94 @@ def barchart(df, var, time_filter_info, data_filter_info, normalize, si_ip): for month_num in range(1, 13): if month_num in available_months_set: - query = f"month=={str(month_num)} and ({filter_var}>={min_val} and {filter_var}<={max_val})" - a = new_df.query(query)[Variables.DOY.col_name].count() - month_in.append(a) - query = f"month=={str(month_num)} and ({filter_var}<{min_val})" - b = new_df.query(query)[Variables.DOY.col_name].count() - month_below.append(b) - query = f"month=={str(month_num)} and {filter_var}>{max_val}" - c = new_df.query(query)[Variables.DOY.col_name].count() - month_above.append(c) + # Calculate for unfiltered data + month_unfiltered = df_unfiltered[ + df_unfiltered[Variables.MONTH.col_name] == month_num + ] + if len(month_unfiltered) > 0: + query = f"month=={str(month_num)} and ({filter_var}>={min_val} and {filter_var}<={max_val})" + a = month_unfiltered.query(query)[Variables.DOY.col_name].count() + month_in.append(a) + query = f"month=={str(month_num)} and ({filter_var}<{min_val})" + b = month_unfiltered.query(query)[Variables.DOY.col_name].count() + month_below.append(b) + query = f"month=={str(month_num)} and {filter_var}>{max_val}" + c = month_unfiltered.query(query)[Variables.DOY.col_name].count() + month_above.append(c) + else: + month_in.append(0) + month_below.append(0) + month_above.append(0) + + # Calculate for filtered data (using original values) + if has_filtered_data(df_filtered) and use_original_for_filtered: + month_filtered = df_filtered[ + df_filtered[Variables.MONTH.col_name] == month_num + ] + if len(month_filtered) > 0: + filtered_var_col = original_var_col + query = f"month=={str(month_num)} and ({filtered_var_col}>={min_val} and {filtered_var_col}<={max_val})" + a = month_filtered.query(query)[Variables.DOY.col_name].count() + month_in_filtered.append(a) + query = ( + f"month=={str(month_num)} and ({filtered_var_col}<{min_val})" + ) + b = month_filtered.query(query)[Variables.DOY.col_name].count() + month_below_filtered.append(b) + query = f"month=={str(month_num)} and {filtered_var_col}>{max_val}" + c = month_filtered.query(query)[Variables.DOY.col_name].count() + month_above_filtered.append(c) + else: + month_in_filtered.append(0) + month_below_filtered.append(0) + month_above_filtered.append(0) + else: + month_in_filtered.append(0) + month_below_filtered.append(0) + month_above_filtered.append(0) else: # No data for this month, append zeros month_in.append(0) month_below.append(0) month_above.append(0) + month_in_filtered.append(0) + month_below_filtered.append(0) + month_above_filtered.append(0) go.Figure() month_names = month_lst + data = [] + + # Add filtered data traces (gray) if any filtered data exists + if ( + has_filter_marker + and filtered_mask is not None + and filtered_mask.any() + and any(month_in_filtered + month_below_filtered + month_above_filtered) + ): + trace1_filtered = go.Bar( + x=month_names, + y=month_in_filtered, + name="IN range (Filtered)", + marker_color="gray", + ) + trace2_filtered = go.Bar( + x=month_names, + y=month_below_filtered, + name="BELOW range (Filtered)", + marker_color="lightgray", + ) + trace3_filtered = go.Bar( + x=month_names, + y=month_above_filtered, + name="ABOVE range (Filtered)", + marker_color="silver", + ) + data = [trace2_filtered, trace1_filtered, trace3_filtered] + + # Add unfiltered data traces (normal colors) trace1 = go.Bar(x=month_names, y=month_in, name="IN range", marker_color=color_in) trace2 = go.Bar( x=month_names, @@ -1034,7 +1456,7 @@ def barchart(df, var, time_filter_info, data_filter_info, normalize, si_ip): name="ABOVE range", marker_color=color_above, ) - data = [trace2, trace1, trace3] + data = data + [trace2, trace1, trace3] fig = go.Figure(data=data) fig.update_layout(barmode="stack", dragmode=False) diff --git a/pages/lib/utils.py b/pages/lib/utils.py index 5bd8cf1..1611d35 100644 --- a/pages/lib/utils.py +++ b/pages/lib/utils.py @@ -394,3 +394,88 @@ def get_time_filter_from_store( state["invert_month"], state["invert_hour"], ) + + +def separate_filtered_data(df, var=None): + # Check if there's a filter marker + has_filter_marker = "_is_filtered" in df.columns + filtered_mask = None + if has_filter_marker: + filtered_mask = df["_is_filtered"] + + # Get original values if available + original_var_col = None + use_original_for_filtered = False + if var is not None: + original_var_col = f"_{var}_original" + use_original_for_filtered = has_filter_marker and original_var_col in df.columns + + # Separate filtered and unfiltered data + if has_filter_marker and filtered_mask is not None: + df_unfiltered = df[~filtered_mask].copy() + df_filtered = df[filtered_mask].copy() if filtered_mask.any() else None + else: + df_unfiltered = df + df_filtered = None + + return { + "has_filter_marker": has_filter_marker, + "filtered_mask": filtered_mask, + "df_unfiltered": df_unfiltered, + "df_filtered": df_filtered, + "original_var_col": original_var_col, + "use_original_for_filtered": use_original_for_filtered, + } + + +def has_filtered_data(df_filtered): + return df_filtered is not None and len(df_filtered) > 0 + + +def get_variable_info(var, si_ip): + variable = VariableInfo.from_col_name(var) + return { + "var_unit": variable.get_unit(si_ip), + "var_range": variable.get_range(si_ip), + "var_name": variable.get_name(), + "var_color": variable.get_color(), + } + + +def unpack_variable_info(var_info, keys=None): + if keys is None: + keys = ["var_unit", "var_range", "var_name", "var_color"] + return tuple(var_info[key] for key in keys) + + +def get_variable_range( + var, df, global_local, si_ip, use_original_for_range=False, original_values=None +): + var_info = get_variable_info(var, si_ip) + var_range = var_info["var_range"] + + if global_local == "global": + return var_range + else: + if use_original_for_range and original_values is not None: + data_max, data_min = get_max_min_value(original_values) + else: + data_max, data_min = get_max_min_value(df[var]) + return [data_min, data_max] + + +def get_original_column_values(df, var): + original_col = f"_{var}_original" + if original_col in df.columns: + return df[original_col].copy() + else: + return df[var].copy() + + +def calculate_daily_statistics(df, var_col, date_col=Variables.UTC_TIME.col_name): + if len(df) == 0: + return pd.DataFrame({"min": [], "max": [], "mean": []}) + + df_with_date = df.copy() + df_with_date["_date"] = df_with_date[date_col].dt.date + return df_with_date.groupby("_date")[var_col].agg(["min", "max", "mean"]) diff --git a/pages/natural_ventilation.py b/pages/natural_ventilation.py index 613bf76..7bded9d 100644 --- a/pages/natural_ventilation.py +++ b/pages/natural_ventilation.py @@ -25,6 +25,7 @@ generate_custom_inputs_nv, determine_month_and_hour_filter, title_with_link, + separate_filtered_data, ) @@ -232,9 +233,9 @@ def nv_heatmap( get_global_filter_state, ) - df = apply_global_month_hour_filter( - df, global_filter_data, Variables.DBT.col_name - ) + # Ensure DBT and DPT are included for filtering + target_columns = [Variables.DBT.col_name, Variables.DPT.col_name] + df = apply_global_month_hour_filter(df, global_filter_data, target_columns) filter_state = get_global_filter_state(global_filter_data) month_range = filter_state["month_range"] @@ -306,35 +307,147 @@ def nv_heatmap( if dpt_data_filter: title += f" and when the {filter_name} is below {max_dpt_val} {filter_unit}." - fig = go.Figure( - data=go.Heatmap( - y=df[Variables.HOUR.col_name] - - 0.5, # Offset by 0.5 to center the hour labels - x=df[Variables.UTC_TIME.col_name].dt.date, - z=df[var], - colorscale=var_color, - zmin=range_z[0], - zmax=range_z[1], - connectgaps=False, - hoverongaps=False, - customdata=np.stack( - (df[Variables.MONTH_NAMES.col_name], df[Variables.DAY.col_name]), - axis=-1, - ), - hovertemplate=( - "" - + var - + ": %{z:.2f} " - + var_unit - + "
" - + "Month: %{customdata[0]}
" - + "Day: %{customdata[1]}
" - + "Hour: %{y}:00
" - ), - colorbar=dict(title=var_unit), - name="", + # Separate filtered and unfiltered data using utility function + filter_info = separate_filtered_data(df, var) + has_filter_marker = filter_info["has_filter_marker"] + filtered_mask = filter_info["filtered_mask"] + original_var_col = filter_info["original_var_col"] + use_original_for_filtered = filter_info["use_original_for_filtered"] + + fig = go.Figure() + + # Add filtered data trace (gray) if any filtered data exists + # Only show gray where there is actual data (not None), not in blank areas + if has_filter_marker and filtered_mask is not None and filtered_mask.any(): + if use_original_for_filtered: + # Use original DBT values for filtered data + filtered_values = df[original_var_col].copy() + # Apply DBT filter to original values to check if they're in range + if dbt_data_filter and (min_dbt_val <= max_dbt_val): + # Only show gray where original DBT is in range + in_range_mask = (filtered_values >= min_dbt_val) & ( + filtered_values <= max_dbt_val + ) + # Also check if DPT filter applies + if dpt_data_filter: + original_filter_var_col = f"_{filter_var}_original" + if original_filter_var_col in df.columns: + dpt_values = df[original_filter_var_col] + in_range_mask = ( + in_range_mask + & (dpt_values >= -200) + & (dpt_values <= max_dpt_val) + ) + else: + dpt_values = df[filter_var] + in_range_mask = ( + in_range_mask + & (dpt_values >= -200) + & (dpt_values <= max_dpt_val) + ) + filtered_values[~in_range_mask] = None + else: + filtered_values = df[var].copy() + + # Only show gray for filtered data points + filtered_values[~filtered_mask] = None + + # Only add trace if there are any valid filtered values + if filtered_values.notna().any(): + fig.add_trace( + go.Heatmap( + y=df[Variables.HOUR.col_name] - 0.5, + x=df[Variables.UTC_TIME.col_name].dt.date, + z=filtered_values, + colorscale=[[0, "lightgray"], [1, "gray"]], + zmin=range_z[0], + zmax=range_z[1], + showscale=False, + connectgaps=False, + hoverongaps=False, + customdata=np.stack( + ( + df[Variables.MONTH_NAMES.col_name], + df[Variables.DAY.col_name], + ), + axis=-1, + ), + hovertemplate=( + "Filtered Data
" + + var + + ": %{z:.2f} " + + var_unit + + "
" + + "Month: %{customdata[0]}
" + + "Day: %{customdata[1]}
" + + "Hour: %{y}:00
" + ), + name="filtered", + ) + ) + + # Add unfiltered data trace (normal color) + base_values = df[var].copy() + base_values[filtered_mask] = None + + fig.add_trace( + go.Heatmap( + y=df[Variables.HOUR.col_name] - 0.5, + x=df[Variables.UTC_TIME.col_name].dt.date, + z=base_values, + colorscale=var_color, + zmin=range_z[0], + zmax=range_z[1], + connectgaps=False, + hoverongaps=False, + customdata=np.stack( + (df[Variables.MONTH_NAMES.col_name], df[Variables.DAY.col_name]), + axis=-1, + ), + hovertemplate=( + "" + + var + + ": %{z:.2f} " + + var_unit + + "
" + + "Month: %{customdata[0]}
" + + "Day: %{customdata[1]}
" + + "Hour: %{y}:00
" + ), + colorbar=dict(title=var_unit), + name="", + ) + ) + else: + # No filtered data, use normal heatmap + fig.add_trace( + go.Heatmap( + y=df[Variables.HOUR.col_name] - 0.5, + x=df[Variables.UTC_TIME.col_name].dt.date, + z=df[var], + colorscale=var_color, + zmin=range_z[0], + zmax=range_z[1], + connectgaps=False, + hoverongaps=False, + customdata=np.stack( + (df[Variables.MONTH_NAMES.col_name], df[Variables.DAY.col_name]), + axis=-1, + ), + hovertemplate=( + "" + + var + + ": %{z:.2f} " + + var_unit + + "
" + + "Month: %{customdata[0]}
" + + "Day: %{customdata[1]}
" + + "Hour: %{y}:00
" + ), + colorbar=dict(title=var_unit), + name="", + ) ) - ) fig.update_layout( template=template, @@ -426,14 +539,40 @@ def nv_bar_chart( df[Variables.NV_ALLOWED.col_name] = 1 + # Store original data info before applying global filter (to know which months originally had data) + if global_filter_data and global_filter_data.get("filter_active", False): + # Create a copy to check which months have data after DBT/DPT filtering (but before global filter) + df_temp = df.copy() + df_temp[Variables.NV_ALLOWED.col_name] = 1 + + # Apply DBT/DPT filters to the temporary copy + if dbt_data_filter and (min_dbt_val <= max_dbt_val): + df_temp.loc[ + (df_temp[var] < min_dbt_val) | (df_temp[var] > max_dbt_val), + Variables.NV_ALLOWED.col_name, + ] = 0 + + if dpt_data_filter: + df_temp.loc[ + (df_temp[filter_var] > max_dpt_val), Variables.NV_ALLOWED.col_name + ] = 0 + + # Check which months have data (NV_ALLOWED > 0) after DBT/DPT filtering + months_with_nv = df_temp[df_temp[Variables.NV_ALLOWED.col_name] > 0] + if len(months_with_nv) > 0: + set(months_with_nv[Variables.UTC_TIME.col_name].dt.month.unique()) + if global_filter_data and global_filter_data.get("filter_active", False): from pages.lib.layout import ( apply_global_month_hour_filter, get_global_filter_state, ) + # Include DBT and DPT in target_columns to preserve original values for filtered data + # Note: Do NOT include NV_ALLOWED in target_columns, as it will be set to None by time_filtering + # for filtered months, which would break the calculation of n_hours_nv_allowed_filtered df = apply_global_month_hour_filter( - df, global_filter_data, Variables.NV_ALLOWED.col_name + df, global_filter_data, [Variables.DBT.col_name, Variables.DPT.col_name] ) filter_state = get_global_filter_state(global_filter_data) @@ -449,49 +588,207 @@ def nv_bar_chart( # Use default values when global filter is not active start_month, end_month, start_hour, end_hour = 1, 12, 0, 24 - # this should be the total after filtering by time - tot_month_hours = ( - df.groupby(df[Variables.UTC_TIME.col_name].dt.month)[ - Variables.NV_ALLOWED.col_name - ] - .sum() - .values - ) - + # Separate filtered and unfiltered data using utility function + filter_info = separate_filtered_data(df, Variables.DBT.col_name) + has_filter_marker = filter_info["has_filter_marker"] + filtered_mask = filter_info["filtered_mask"] + df_unfiltered = filter_info["df_unfiltered"] + df_filtered = filter_info["df_filtered"] + + # Calculate total hours per month (for both filtered and unfiltered) - ensure all 12 months are included + # This should be calculated BEFORE applying DBT/DPT filters, as it represents total hours in the selected time range + tot_month_hours_unfiltered = np.zeros(12) + tot_unfiltered_grouped = df_unfiltered.groupby( + df_unfiltered[Variables.UTC_TIME.col_name].dt.month + )[Variables.NV_ALLOWED.col_name].sum() + for month_idx in range(1, 13): + if month_idx in tot_unfiltered_grouped.index: + tot_month_hours_unfiltered[month_idx - 1] = tot_unfiltered_grouped[ + month_idx + ] + + # Calculate total filtered hours BEFORE applying DBT/DPT filters + # This represents all hours that were filtered by the global month/hour filter + # Simply count the number of rows in df_filtered for each month (each row = 1 hour) + tot_month_hours_filtered = np.zeros(12) + if df_filtered is not None and len(df_filtered) > 0: + # Count rows per month (each row represents 1 hour) + # This is the most reliable way as it doesn't depend on NV_ALLOWED being set + tot_filtered_grouped = df_filtered.groupby( + df_filtered[Variables.UTC_TIME.col_name].dt.month + ).size() + for month_idx in range(1, 13): + if month_idx in tot_filtered_grouped.index: + tot_month_hours_filtered[month_idx - 1] = tot_filtered_grouped[ + month_idx + ] + + # Apply DBT and DPT filters to unfiltered data if dbt_data_filter and (min_dbt_val <= max_dbt_val): - df.loc[ - (df[var] < min_dbt_val) | (df[var] > max_dbt_val), + df_unfiltered.loc[ + (df_unfiltered[var] < min_dbt_val) | (df_unfiltered[var] > max_dbt_val), Variables.NV_ALLOWED.col_name, ] = 0 if dpt_data_filter: - df.loc[(df[filter_var] > max_dpt_val), Variables.NV_ALLOWED.col_name] = 0 + df_unfiltered.loc[ + (df_unfiltered[filter_var] > max_dpt_val), Variables.NV_ALLOWED.col_name + ] = 0 - n_hours_nv_allowed = ( - df.dropna(subset=Variables.NV_ALLOWED.col_name) - .groupby(df[Variables.UTC_TIME.col_name].dt.month)[ + # Apply DBT and DPT filters to filtered data (using original values if available) + if df_filtered is not None and len(df_filtered) > 0: + original_var_col = f"_{var}_original" + original_filter_var_col = f"_{filter_var}_original" + use_original_var = original_var_col in df_filtered.columns + use_original_filter_var = original_filter_var_col in df_filtered.columns + + if dbt_data_filter and (min_dbt_val <= max_dbt_val): + filter_var_to_use = original_var_col if use_original_var else var + df_filtered.loc[ + (df_filtered[filter_var_to_use] < min_dbt_val) + | (df_filtered[filter_var_to_use] > max_dbt_val), + Variables.NV_ALLOWED.col_name, + ] = 0 + + if dpt_data_filter: + filter_var_to_use = ( + original_filter_var_col if use_original_filter_var else filter_var + ) + df_filtered.loc[ + (df_filtered[filter_var_to_use] > max_dpt_val), + Variables.NV_ALLOWED.col_name, + ] = 0 + + # Calculate hours for unfiltered data - ensure all 12 months are included + n_hours_nv_allowed_unfiltered = np.zeros(12) + n_hours_unfiltered_grouped = ( + df_unfiltered.dropna(subset=Variables.NV_ALLOWED.col_name) + .groupby(df_unfiltered[Variables.UTC_TIME.col_name].dt.month)[ Variables.NV_ALLOWED.col_name ] .sum() - .values ) + for month_idx in range(1, 13): + if month_idx in n_hours_unfiltered_grouped.index: + n_hours_nv_allowed_unfiltered[month_idx - 1] = n_hours_unfiltered_grouped[ + month_idx + ] + + # Calculate hours for filtered data - ensure all 12 months are included + n_hours_nv_allowed_filtered = np.zeros(12) + if df_filtered is not None and len(df_filtered) > 0: + n_hours_filtered_grouped = ( + df_filtered.dropna(subset=Variables.NV_ALLOWED.col_name) + .groupby(df_filtered[Variables.UTC_TIME.col_name].dt.month)[ + Variables.NV_ALLOWED.col_name + ] + .sum() + ) + for month_idx in range(1, 13): + if month_idx in n_hours_filtered_grouped.index: + n_hours_nv_allowed_filtered[month_idx - 1] = n_hours_filtered_grouped[ + month_idx + ] + + # Calculate percentages - handle division by zero + per_time_nv_allowed_unfiltered = np.zeros(12) + for i in range(12): + if tot_month_hours_unfiltered[i] > 0: + per_time_nv_allowed_unfiltered[i] = np.round( + 100 * (n_hours_nv_allowed_unfiltered[i] / tot_month_hours_unfiltered[i]) + ) - per_time_nv_allowed = np.round(100 * (n_hours_nv_allowed / tot_month_hours)) + per_time_nv_allowed_filtered = np.zeros(12) + # Calculate percentages for all months where filtered hours exist + # Even if nv_allowed is 0, we should still show the gray bar (with 0% value) + for i in range(12): + if tot_month_hours_filtered[i] > 0: + per_time_nv_allowed_filtered[i] = np.round( + 100 * (n_hours_nv_allowed_filtered[i] / tot_month_hours_filtered[i]) + ) - if not normalize: - fig = go.Figure( - go.Bar( - x=df[Variables.MONTH_NAMES.col_name].unique(), - y=n_hours_nv_allowed, - name="", - marker_color=color_in, - customdata=np.stack((n_hours_nv_allowed, per_time_nv_allowed), axis=-1), + month_names = month_lst # Use month_lst to ensure all 12 months are included + traces = [] + + # Add filtered data traces (gray) if any filtered data exists + # For normalize mode: Show gray bars for months that have data but are outside the global filter range + # For non-normalize mode: Show gray bars for all filtered months + has_filtered_data = False + if has_filter_marker and filtered_mask is not None and filtered_mask.any(): + # Show gray bars if there are any filtered hours in any month + has_filtered_data = np.any(tot_month_hours_filtered > 0) + + if has_filtered_data: + if not normalize: + trace_filtered = go.Bar( + x=month_names, + y=n_hours_nv_allowed_filtered, + name="Natural Ventilation (Filtered)", + marker_color="gray", + customdata=np.stack( + (n_hours_nv_allowed_filtered, per_time_nv_allowed_filtered), axis=-1 + ), hovertemplate=( - "natural ventilation possible for:
%{customdata[0]} hrs or" + "Filtered Data
natural ventilation possible for:
%{customdata[0]} hrs or" "
%{customdata[1]}% of selected time
" ), ) + traces.append(trace_filtered) + else: + # For normalize mode: Show gray bars for months outside the global filter range + # Use actual percentage values, but for 0% values, use a minimal visible height (0.1%) + # so users can see that these months have filtered data + per_time_display_filtered = per_time_nv_allowed_filtered.copy() + + # Set None for months without filtered data (so they don't show gray bars) + # For months with filtered data but 0% NV, use 0.1% for minimal visibility + for i in range(12): + if tot_month_hours_filtered[i] == 0: + per_time_display_filtered[i] = None + elif per_time_nv_allowed_filtered[i] == 0: + # Use 0.1% for months with filtered data but 0% NV (very small but visible) + per_time_display_filtered[i] = 0.1 + + trace_filtered = go.Bar( + x=month_names, + y=per_time_display_filtered, + name="Natural Ventilation (Filtered)", + marker_color="gray", + marker_line_color="gray", + marker_line_width=1, + customdata=np.stack( + ( + n_hours_nv_allowed_filtered, + per_time_nv_allowed_filtered, + tot_month_hours_filtered, + ), + axis=-1, + ), + hovertemplate=( + "Filtered Data
natural ventilation possible for:
%{customdata[0]} hrs or
%{" + "customdata[1]:.2f}% of filtered time range
Total filtered hours: %{customdata[2]:.0f}
" + ), + base=0, + opacity=0.8, + ) + traces.append(trace_filtered) + + # Add unfiltered data traces (normal colors) + if not normalize: + trace_unfiltered = go.Bar( + x=month_names, + y=n_hours_nv_allowed_unfiltered, + name="Natural Ventilation", + marker_color=color_in, + customdata=np.stack( + (n_hours_nv_allowed_unfiltered, per_time_nv_allowed_unfiltered), axis=-1 + ), + hovertemplate=( + "natural ventilation possible for:
%{customdata[0]} hrs or" + "
%{customdata[1]}% of selected time
" + ), ) + traces.append(trace_unfiltered) title = ( f"Number of hours the {var_name}" @@ -499,22 +796,21 @@ def nv_bar_chart( + " to " + f" {max_dbt_val} {var_unit}" ) - fig.update_yaxes(title_text="hours", range=[0, 744]) - else: - trace1 = go.Bar( - x=df[Variables.MONTH_NAMES.col_name].unique(), - y=per_time_nv_allowed, - name="", + trace_unfiltered = go.Bar( + x=month_names, + y=per_time_nv_allowed_unfiltered, + name="Natural Ventilation", marker_color=color_in, - customdata=np.stack((n_hours_nv_allowed, per_time_nv_allowed), axis=-1), + customdata=np.stack( + (n_hours_nv_allowed_unfiltered, per_time_nv_allowed_unfiltered), axis=-1 + ), hovertemplate=( "natural ventilation possible for:
%{customdata[0]} hrs or
%{" "customdata[1]}% of selected time
" ), ) - - fig = go.Figure(data=trace1) + traces.append(trace_unfiltered) title = ( f"Percentage of hours the {var_name}" @@ -522,6 +818,12 @@ def nv_bar_chart( + f" to {max_dbt_val}" + f" {var_unit}" ) + + fig = go.Figure(data=traces) + + if not normalize: + fig.update_yaxes(title_text="hours", range=[0, 744]) + else: fig.update_yaxes(title_text="Percentage (%)", range=[0, 100]) if global_filter_data and global_filter_data.get("filter_active", False): @@ -533,12 +835,15 @@ def nv_bar_chart( if dpt_data_filter: title += f" when the {filter_name} is below {max_dpt_val} {filter_unit}." + # Use barmode="relative" to show filtered and unfiltered bars side by side + # Only use relative mode if we have filtered data (non-normalize mode only) fig.update_layout( template=template, title=title, barnorm="", dragmode=False, margin=tight_margins.copy().update({"t": 55}), + barmode="relative" if has_filtered_data else "group", ) fig.update_xaxes( diff --git a/pages/outdoor.py b/pages/outdoor.py index 8f2ec39..fb7805a 100644 --- a/pages/outdoor.py +++ b/pages/outdoor.py @@ -298,7 +298,7 @@ def update_tab_utci_category( title="Thermal stress", titleside="top", tickmode="array", - tickvals=np.linspace(4.75, -4.75, 10), + tickvals=np.linspace(4, -5, 10), ticktext=[ "extreme heat stress", "very strong heat stress", @@ -340,7 +340,11 @@ def update_tab_utci_summary_chart(var, normalize, global_filter_data, df, meta, if global_filter_data and global_filter_data.get("filter_active", False): from pages.lib.layout import apply_global_month_hour_filter - df = apply_global_month_hour_filter(df, global_filter_data, var) + # Include both the original UTCI variable and the categories column + # so we can preserve original values for filtered data + df = apply_global_month_hour_filter( + df, global_filter_data, [var, var + "_categories"] + ) # Unified filter state for both active and inactive cases time_filter, month, hour, invert_month, invert_hour = get_time_filter_from_store( diff --git a/pages/psy-chart.py b/pages/psy-chart.py index 7f94a25..92fccc0 100644 --- a/pages/psy-chart.py +++ b/pages/psy-chart.py @@ -10,7 +10,7 @@ from pythermalcomfort import psychrometrics as psy from config import PageUrls, DocLinks, PageInfo, UnitSystem -from pages.lib.utils import get_max_min_value +from pages.lib.utils import get_max_min_value, separate_filtered_data from pages.lib.global_element_ids import ElementIds from pages.lib.global_variables import Variables, VariableInfo from pages.lib.global_id_buttons import IdButtons @@ -34,6 +34,13 @@ ) +def _safe_get_column(df, column_name, default_value=0): + if column_name in df.columns: + return df[column_name] + else: + return [default_value] * len(df) + + dash.register_page( __name__, name=PageInfo.PSYCHROMETRIC_NAME, @@ -177,7 +184,14 @@ def update_psych_chart( get_global_filter_state, ) - df = apply_global_month_hour_filter(df, global_filter_data) + # Determine which columns to filter - need DBT and HR at minimum, plus colorby_var if it's not None/Frequency + target_columns = [Variables.DBT.col_name, Variables.HR.col_name] + if colorby_var not in ["None", "Frequency"]: + target_columns.append(colorby_var) + if data_filter and data_filter_var: + target_columns.append(data_filter_var) + + df = apply_global_month_hour_filter(df, global_filter_data, target_columns) filter_state = get_global_filter_state(global_filter_data) month_range = filter_state["month_range"] @@ -286,117 +300,126 @@ def update_psych_chart( ) ) - df_hr_multiply = list(df[Variables.HR.col_name]) - for k in range(len(df_hr_multiply)): - df_hr_multiply[k] = df_hr_multiply[k] * 1000 - if var == "None": - fig.add_trace( - go.Scatter( - x=df[Variables.DBT.col_name], - y=df_hr_multiply, - showlegend=False, - mode="markers", - marker=dict( - size=6, - color=var_color, - showscale=False, - opacity=0.2, - ), - hovertemplate=VariableInfo.from_col_name( - Variables.DBT.col_name - ).get_name() - + ": %{x:.2f}" - + VariableInfo.from_col_name(Variables.DBT.col_name).get_name(), - name="", + # Separate filtered and unfiltered data using utility function + # Note: psy-chart needs to check multiple original columns (DBT, HR, and var) + filter_info = separate_filtered_data(df, Variables.DBT.col_name) + df_unfiltered = filter_info["df_unfiltered"] + + # Process HR for unfiltered data + df_unfiltered_hr_multiply = list(df_unfiltered[Variables.HR.col_name]) + for k in range(len(df_unfiltered_hr_multiply)): + df_unfiltered_hr_multiply[k] = df_unfiltered_hr_multiply[k] * 1000 + + # Filtered data traces removed - no gray filtering effect for psychrometric chart + + # Add unfiltered data traces (normal colors) + if len(df_unfiltered) > 0: + if var == "None": + fig.add_trace( + go.Scatter( + x=df_unfiltered[Variables.DBT.col_name], + y=df_unfiltered_hr_multiply, + showlegend=False, + mode="markers", + marker=dict( + size=6, + color=var_color, + showscale=False, + opacity=0.2, + ), + hovertemplate=VariableInfo.from_col_name( + Variables.DBT.col_name + ).get_name() + + ": %{x:.2f}" + + VariableInfo.from_col_name(Variables.DBT.col_name).get_unit( + si_ip + ), + name="", + ) ) - ) - elif var == "Frequency": - fig.add_trace( - go.Histogram2d( - x=df[Variables.DBT.col_name], - y=df_hr_multiply, - name="", - colorscale=var_color, - hovertemplate="", - autobinx=False, - xbins=dict(start=-50, end=100, size=1), + elif var == "Frequency": + fig.add_trace( + go.Histogram2d( + x=df_unfiltered[Variables.DBT.col_name], + y=df_unfiltered_hr_multiply, + name="", + colorscale=var_color, + hovertemplate="", + autobinx=False, + xbins=dict(start=-50, end=100, size=1), + ) ) - ) - # fig.add_trace( - # go.Scatter( - # x=dbt_list, - # y=rh_df["rh100"], - # showlegend=False, - # mode="none", - # name="", - # fill="toself", - # fillcolor="#fff", - # ) - # ) + # Filtered data removed - no gray filtering effect for psychrometric chart - else: - var_colorbar = dict( - thickness=30, - title=var_unit + "
", - ) - - if var_unit == "Thermal stress": - var_colorbar["tickvals"] = [4, 3, 2, 1, 0, -1, -2, -3, -4, -5] - var_colorbar["ticktext"] = [ - "extreme heat stress", - "very strong heat stress", - "strong heat stress", - "moderate heat stress", - "no thermal stress", - "slight cold stress", - "moderate cold stress", - "strong cold stress", - "very strong cold stress", - "extreme cold stress", - ] + else: + var_colorbar = dict( + thickness=30, + title=var_unit + "
", + ) - fig.add_trace( - go.Scatter( - x=df[Variables.DBT.col_name], - y=df_hr_multiply, - showlegend=False, - mode="markers", - marker=dict( - size=5, - color=df[var], - showscale=True, - opacity=0.3, - colorscale=var_color, - colorbar=var_colorbar, - ), - customdata=np.stack( - (df[Variables.RH.col_name], df["h"], df[var], df["t_dp"]), axis=-1 - ), - hovertemplate=VariableInfo.from_col_name( - Variables.DBT.col_name - ).get_name() - + ": %{x:.2f}" - + VariableInfo.from_col_name(Variables.DBT.col_name).get_unit(si_ip) - + "
" - + VariableInfo.from_col_name(Variables.RH.col_name).get_name() - + ": %{customdata[0]:.2f}" - + VariableInfo.from_col_name(Variables.RH.col_name).get_unit(si_ip) - + "
" - + VariableInfo.from_col_name("h").get_name() - + ": %{customdata[1]:.2f}" - + VariableInfo.from_col_name("h").get_unit(si_ip) - + "
" - + VariableInfo.from_col_name("t_dp").get_name() - + ": %{customdata[3]:.2f}" - + VariableInfo.from_col_name("t_dp").get_unit(si_ip) - + "
" - + "
" - + var_name - + ": %{customdata[2]:.2f}" - + var_unit, - name="", + if var_unit == "Thermal stress": + var_colorbar["tickvals"] = [4, 3, 2, 1, 0, -1, -2, -3, -4, -5] + var_colorbar["ticktext"] = [ + "extreme heat stress", + "very strong heat stress", + "strong heat stress", + "moderate heat stress", + "no thermal stress", + "slight cold stress", + "moderate cold stress", + "strong cold stress", + "very strong cold stress", + "extreme cold stress", + ] + + fig.add_trace( + go.Scatter( + x=df_unfiltered[Variables.DBT.col_name], + y=df_unfiltered_hr_multiply, + showlegend=False, + mode="markers", + marker=dict( + size=5, + color=df_unfiltered[var], + showscale=True, + opacity=0.3, + colorscale=var_color, + colorbar=var_colorbar, + ), + customdata=np.stack( + ( + df_unfiltered[Variables.RH.col_name], + df_unfiltered["h"], + df_unfiltered[var], + df_unfiltered["t_dp"], + ), + axis=-1, + ), + hovertemplate=VariableInfo.from_col_name( + Variables.DBT.col_name + ).get_name() + + ": %{x:.2f}" + + VariableInfo.from_col_name(Variables.DBT.col_name).get_unit(si_ip) + + "
" + + VariableInfo.from_col_name(Variables.RH.col_name).get_name() + + ": %{customdata[0]:.2f}" + + VariableInfo.from_col_name(Variables.RH.col_name).get_unit(si_ip) + + "
" + + VariableInfo.from_col_name("h").get_name() + + ": %{customdata[1]:.2f}" + + VariableInfo.from_col_name("h").get_unit(si_ip) + + "
" + + VariableInfo.from_col_name("t_dp").get_name() + + ": %{customdata[3]:.2f}" + + VariableInfo.from_col_name("t_dp").get_unit(si_ip) + + "
" + + "
" + + var_name + + ": %{customdata[2]:.2f}" + + var_unit, + name="", + ) ) - ) xtitle_name = ( "Temperature" diff --git a/pages/summary.py b/pages/summary.py index 84d2064..9b11e94 100644 --- a/pages/summary.py +++ b/pages/summary.py @@ -321,31 +321,100 @@ def degree_day_chart( color_hdd = "red" color_cdd = "dodgerblue" + # Check if there's a filter marker + has_filter_marker = "_is_filtered" in df.columns + filtered_mask = None + if has_filter_marker: + filtered_mask = df["_is_filtered"] + + # Get original DBT values if available + original_dbt_col = f"_{Variables.DBT.col_name}_original" + use_original_for_filtered = has_filter_marker and original_dbt_col in df.columns + hdd_array, cdd_array = [], [] + hdd_array_filtered, cdd_array_filtered = [], [] months = df[Variables.MONTH_NAMES.col_name].unique() for i in range(1, 13): query_month = "month==" + month_query = query_month + str(i) + month_df = df.query(month_query) - a = df.query(query_month + str(i) + " and DBT<=" + str(hdd_setpoint))[ - Variables.DBT.col_name - ].sub(hdd_setpoint) - hdd_array.append(int(a.sum(skipna=True) / 24)) + # Calculate HDD and CDD for unfiltered data + if has_filter_marker and filtered_mask is not None: + unfiltered_mask = ~month_df["_is_filtered"] + unfiltered_dbt = month_df[Variables.DBT.col_name][unfiltered_mask] + else: + unfiltered_dbt = month_df[Variables.DBT.col_name] - a = df.query(query_month + str(i) + " and DBT>=" + str(cdd_setpoint))[ - Variables.DBT.col_name - ].sub(cdd_setpoint) - cdd_array.append(int(a.sum(skipna=True) / 24)) + # Calculate HDD for unfiltered data + a_unfiltered_hdd = unfiltered_dbt[unfiltered_dbt <= hdd_setpoint].sub( + hdd_setpoint + ) + hdd_array.append(int(a_unfiltered_hdd.sum(skipna=True) / 24)) - trace1 = go.Bar( - x=months, - y=hdd_array, - name="Heating Degree Days", - marker_color=color_hdd, - customdata=[abs(x) for x in hdd_array], - hovertemplate=" Heating Degree Days:
%{customdata} per month
", - ) + # Calculate CDD for unfiltered data + a_unfiltered_cdd = unfiltered_dbt[unfiltered_dbt >= cdd_setpoint].sub( + cdd_setpoint + ) + cdd_array.append(int(a_unfiltered_cdd.sum(skipna=True) / 24)) + + # Calculate HDD and CDD for filtered data (if any) + if ( + has_filter_marker + and filtered_mask is not None + and month_df["_is_filtered"].any() + ): + filtered_mask_month = month_df["_is_filtered"] + + if use_original_for_filtered: + # Use original DBT values for filtered data + month_indices = month_df[filtered_mask_month].index + filtered_dbt = df.loc[month_indices, original_dbt_col] + else: + # Fallback to current DBT values (shouldn't happen if filter is applied correctly) + filtered_dbt = month_df[Variables.DBT.col_name][filtered_mask_month] + + # Calculate HDD for filtered data + a_filtered_hdd = filtered_dbt[filtered_dbt <= hdd_setpoint].sub( + hdd_setpoint + ) + hdd_array_filtered.append(int(a_filtered_hdd.sum(skipna=True) / 24)) + # Calculate CDD for filtered data + a_filtered_cdd = filtered_dbt[filtered_dbt >= cdd_setpoint].sub( + cdd_setpoint + ) + cdd_array_filtered.append(int(a_filtered_cdd.sum(skipna=True) / 24)) + else: + hdd_array_filtered.append(0) + cdd_array_filtered.append(0) + + traces = [] + + # Add filtered data traces (gray) if any filtered data exists + if has_filter_marker and filtered_mask is not None and filtered_mask.any(): + trace_cdd_filtered = go.Bar( + x=months, + y=cdd_array_filtered, + name="Cooling Degree Days (Filtered)", + marker_color="gray", + customdata=cdd_array_filtered, + hovertemplate="Filtered Data
Cooling Degree Days:
%{customdata} per month
", + ) + traces.append(trace_cdd_filtered) + + trace_hdd_filtered = go.Bar( + x=months, + y=hdd_array_filtered, + name="Heating Degree Days (Filtered)", + marker_color="lightgray", + customdata=[abs(x) for x in hdd_array_filtered], + hovertemplate="Filtered Data
Heating Degree Days:
%{customdata} per month
", + ) + traces.append(trace_hdd_filtered) + + # Add unfiltered data traces (normal colors) trace2 = go.Bar( x=months, y=cdd_array, @@ -354,8 +423,19 @@ def degree_day_chart( customdata=cdd_array, hovertemplate="Cooling Degree Days:
%{customdata} per month
", ) + traces.append(trace2) + + trace1 = go.Bar( + x=months, + y=hdd_array, + name="Heating Degree Days", + marker_color=color_hdd, + customdata=[abs(x) for x in hdd_array], + hovertemplate="Heating Degree Days:
%{customdata} per month
", + ) + traces.append(trace1) - fig = go.Figure(data=[trace2, trace1]) + fig = go.Figure(data=traces) fig.update_layout( barmode="relative", margin=tight_margins, diff --git a/pages/sun.py b/pages/sun.py index 2f7c831..0fd6012 100644 --- a/pages/sun.py +++ b/pages/sun.py @@ -213,17 +213,41 @@ def monthly_and_cloud_chart(_, global_filter_data, df, meta, si_ip): Variables.TOT_SKY_COVER.col_name, ], ) - # Filter out the filtered rows for solar radiation calculations - if "_is_filtered" in df.columns: - df = df[~df["_is_filtered"]] - - # Sun Radiation - monthly = monthly_solar(df, si_ip) + # Don't filter out the filtered rows - keep them for gray display + # The monthly_solar and barchart functions will handle filtering + + # Sun Radiation - ensure all necessary columns are included + base_columns = [ + Variables.GLOB_HOR_RAD.col_name, + Variables.DIF_HOR_RAD.col_name, + Variables.MONTH.col_name, + Variables.HOUR.col_name, + Variables.MONTH_NAMES.col_name, + ] + if "_is_filtered" in df.columns: + base_columns.append("_is_filtered") + if f"_{Variables.GLOB_HOR_RAD.col_name}_original" in df.columns: + base_columns.append(f"_{Variables.GLOB_HOR_RAD.col_name}_original") + if f"_{Variables.DIF_HOR_RAD.col_name}_original" in df.columns: + base_columns.append(f"_{Variables.DIF_HOR_RAD.col_name}_original") + monthly = monthly_solar(df[base_columns], si_ip) monthly = monthly.update_layout(margin=tight_margins) - # Cloud Cover + # Cloud Cover - remove filtered columns to disable gray filtering effect + cloud_base_columns = [ + Variables.TOT_SKY_COVER.col_name, + Variables.MONTH.col_name, + Variables.DOY.col_name, + ] + # Create a copy without filtered columns to disable gray filtering + cloud_df = df[cloud_base_columns].copy() cover = barchart( - df, Variables.TOT_SKY_COVER.col_name, [False], [False, "", 3, 7], True, si_ip + cloud_df, + Variables.TOT_SKY_COVER.col_name, + [False], + [False, "", 3, 7], + True, + si_ip, ) cover = cover.update_layout( margin=tight_margins, @@ -282,6 +306,23 @@ def sun_path_chart(_, view, var, global_local, global_filter_data, df, meta, si_ target_cols.append(var) df = apply_global_month_hour_filter(df, global_filter_data, target_cols) + # Ensure all necessary columns are included for filtered data display + base_columns = [ + Variables.APPARENT_ELEVATION.col_name, + Variables.APPARENT_ZENITH.col_name, + Variables.AZIMUTH.col_name, + Variables.ELEVATION.col_name, + Variables.DAY.col_name, + Variables.MONTH_NAMES.col_name, + Variables.HOUR.col_name, + ] + if "_is_filtered" in df.columns: + base_columns.append("_is_filtered") + if var != "None" and f"_{var}_original" in df.columns: + base_columns.append(f"_{var}_original") + if var != "None": + base_columns.append(var) + custom_inputs = "" if var == "None" else f"{var}" units = "" if var == "None" else generate_units(si_ip) if view == "polar": @@ -290,7 +331,7 @@ def sun_path_chart(_, view, var, global_local, global_filter_data, df, meta, si_ config=generate_chart_name( TabNames.SPHERICAL_SUNPATH, meta, custom_inputs, units ), - figure=polar_graph(df, meta, global_local, var, si_ip), + figure=polar_graph(df[base_columns], meta, global_local, var, si_ip), ) else: return dcc.Graph( @@ -298,7 +339,9 @@ def sun_path_chart(_, view, var, global_local, global_filter_data, df, meta, si_ config=generate_chart_name( TabNames.CARTESIAN_SUNPATH, meta, custom_inputs, units ), - figure=custom_cartesian_solar(df, meta, global_local, var, si_ip), + figure=custom_cartesian_solar( + df[base_columns], meta, global_local, var, si_ip + ), ) diff --git a/pages/t_rh.py b/pages/t_rh.py index 24f4dc6..5a6a074 100644 --- a/pages/t_rh.py +++ b/pages/t_rh.py @@ -123,7 +123,26 @@ def update_yearly_chart(_, global_local, dd_value, global_filter_data, df, meta, df = apply_global_month_hour_filter(df, global_filter_data, target_columns) if dd_value == dropdown_names[var_to_plot[0]]: - dbt_yearly = yearly_profile(df, Variables.DBT.col_name, global_local, si_ip) + # Ensure all necessary columns are included for filtered data display + required_cols = [ + Variables.DBT.col_name, + Variables.UTC_TIME.col_name, + Variables.MONTH_NAMES.col_name, + Variables.DAY.col_name, + Variables.DOY.col_name, + Variables.ADAPTIVE_CMF_80_LOW.col_name, + Variables.ADAPTIVE_CMF_80_UP.col_name, + Variables.ADAPTIVE_CMF_90_LOW.col_name, + Variables.ADAPTIVE_CMF_90_UP.col_name, + Variables.ADAPTIVE_CMF_RMT.col_name, + ] + if "_is_filtered" in df.columns: + required_cols.append("_is_filtered") + if f"_{Variables.DBT.col_name}_original" in df.columns: + required_cols.append(f"_{Variables.DBT.col_name}_original") + dbt_yearly = yearly_profile( + df[required_cols], Variables.DBT.col_name, global_local, si_ip + ) dbt_yearly.update_layout(xaxis=dict(rangeslider=dict(visible=True))) units = generate_units_degree(si_ip) return dcc.Graph( @@ -133,7 +152,20 @@ def update_yearly_chart(_, global_local, dd_value, global_filter_data, df, meta, figure=dbt_yearly, ) else: - rh_yearly = yearly_profile(df, Variables.RH.col_name, global_local, si_ip) + # Ensure all necessary columns are included for filtered data display + required_cols = [ + Variables.RH.col_name, + Variables.UTC_TIME.col_name, + Variables.MONTH_NAMES.col_name, + Variables.DAY.col_name, + ] + if "_is_filtered" in df.columns: + required_cols.append("_is_filtered") + if f"_{Variables.RH.col_name}_original" in df.columns: + required_cols.append(f"_{Variables.RH.col_name}_original") + rh_yearly = yearly_profile( + df[required_cols], Variables.RH.col_name, global_local, si_ip + ) rh_yearly.update_layout(xaxis=dict(rangeslider=dict(visible=True))) units = generate_units(si_ip) return dcc.Graph( @@ -164,42 +196,50 @@ def update_daily(_, global_local, dd_value, global_filter_data, df, meta, si_ip) df = apply_global_month_hour_filter(df, global_filter_data, target_columns) if dd_value == dropdown_names[var_to_plot[0]]: + # Ensure all necessary columns are included for filtered data display + base_columns = [ + Variables.DBT.col_name, + Variables.HOUR.col_name, + Variables.UTC_TIME.col_name, + Variables.MONTH_NAMES.col_name, + Variables.DAY.col_name, + Variables.MONTH.col_name, + ] + if "_is_filtered" in df.columns: + base_columns.append("_is_filtered") + if f"_{Variables.DBT.col_name}_original" in df.columns: + base_columns.append(f"_{Variables.DBT.col_name}_original") units = generate_units_degree(si_ip) return dcc.Graph( config=generate_chart_name( TabNames.DRY_BULB_TEMPERATURE_DAILY, meta, units ), figure=daily_profile( - df[ - [ - Variables.DBT.col_name, - Variables.HOUR.col_name, - Variables.UTC_TIME.col_name, - Variables.MONTH_NAMES.col_name, - Variables.DAY.col_name, - Variables.MONTH.col_name, - ] - ], + df[base_columns], Variables.DBT.col_name, global_local, si_ip, ), ) else: + # Ensure all necessary columns are included for filtered data display + base_columns = [ + Variables.RH.col_name, + Variables.HOUR.col_name, + Variables.UTC_TIME.col_name, + Variables.MONTH_NAMES.col_name, + Variables.DAY.col_name, + Variables.MONTH.col_name, + ] + if "_is_filtered" in df.columns: + base_columns.append("_is_filtered") + if f"_{Variables.RH.col_name}_original" in df.columns: + base_columns.append(f"_{Variables.RH.col_name}_original") units = generate_units(si_ip) return dcc.Graph( config=generate_chart_name(TabNames.RELATIVE_HUMIDITY_DAILY, meta, units), figure=daily_profile( - df[ - [ - Variables.RH.col_name, - Variables.HOUR.col_name, - Variables.UTC_TIME.col_name, - Variables.MONTH_NAMES.col_name, - Variables.DAY.col_name, - Variables.MONTH.col_name, - ] - ], + df[base_columns], Variables.RH.col_name, global_local, si_ip, diff --git a/tests/test_filter.py b/tests/test_filter.py index 95a3367..9e377b4 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -130,13 +130,26 @@ def _chart_state_hash(page: Page, chart_selector: str) -> str: return str(hash(html)) +def _wait_dom_change(page: Page, chart_selector: str, prev_html: str): + """ + Wait until the target chart element's innerHTML changes from prev_html. + """ + page.wait_for_function( + "(args) => { const [sel, prev] = args; const el = document.querySelector(sel); return el && el.innerHTML !== prev; }", + arg=[chart_selector, prev_html], + ) + + def assert_chart_changes_by_three_steps(page: Page, chart_selector: str): - """Test chart reactivity across month/hour/both filter changes without screenshots.""" + """Test chart reactivity across month/hour/both filter changes.""" base_hash = _chart_state_hash(page, chart_selector) + base_html = page.locator(chart_selector).first.inner_html() changed = False for months in NARROW_MONTHS: apply_filter(page, months, BASELINE_HOUR) + _wait_dom_change(page, chart_selector, base_html) + base_html = page.locator(chart_selector).first.inner_html() if _chart_state_hash(page, chart_selector) != base_hash: changed = True break @@ -144,6 +157,8 @@ def assert_chart_changes_by_three_steps(page: Page, chart_selector: str): if not changed: for hours in NARROW_HOURS: apply_filter(page, BASELINE_MONTH, hours) + _wait_dom_change(page, chart_selector, base_html) + base_html = page.locator(chart_selector).first.inner_html() if _chart_state_hash(page, chart_selector) != base_hash: changed = True break @@ -151,6 +166,8 @@ def assert_chart_changes_by_three_steps(page: Page, chart_selector: str): if not changed: months, hours = NARROW_MONTHS[0], NARROW_HOURS[0] apply_filter(page, months, hours) + _wait_dom_change(page, chart_selector, base_html) + page.locator(chart_selector).first.inner_html() if _chart_state_hash(page, chart_selector) != base_hash: changed = True diff --git a/tests/test_summary.py b/tests/test_summary.py index d0ce8f2..9faa34f 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -77,12 +77,12 @@ def test_unit_switch(page: Page): """ Verify that the banner radio buttons (SI/IP) correctly toggle. """ - nav_controls = page.locator("#nav-group-controls") - nav_controls.click(force=True) + # nav_controls = page.locator("#nav-group-controls") + # nav_controls.click(force=True) # Click the "IP" option ip_button = page.get_by_text("IP", exact=True) - expect(ip_button).to_be_visible() + expect(ip_button).to_be_enabled() ip_button.scroll_into_view_if_needed() ip_button.wait_for(state="visible") ip_button.click(force=True) diff --git a/tests/test_t_rh.py b/tests/test_t_rh.py index 57c22ea..d8e5c99 100644 --- a/tests/test_t_rh.py +++ b/tests/test_t_rh.py @@ -57,9 +57,6 @@ def test_banner_unit_switch(page: Page): """ Verify that the banner radio buttons (Global/Local) correctly toggle. """ - nav_controls = page.locator("#nav-group-controls") - nav_controls.click(force=True) - # Click the "Global" option global_button = page.get_by_text("Global", exact=True) global_button.scroll_into_view_if_needed()