diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index ce86026..ac315ca 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -1,6 +1,8 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +# Updating to use `uv`. Versions pinned are current as of May 4, 2026. + name: Python application on: @@ -18,22 +20,46 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.13 - uses: actions/setup-python@v3 + - uses: actions/checkout@v6 + + + + - name: "Set up Python" + uses: actions/setup-python@v6 + with: + python-version-file: ".python-version" + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b with: - python-version: "3.13" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest + # Install a specific version of uv. + version: "0.11.8" + + - name: Install the project + run: uv sync --locked --all-extras --dev + + # Broken. + - name: Run tests + # Run test using uv + run: uv run -m tests + + + # OLD, for reference + # - name: Set up Python 3.13 + # uses: actions/setup-python@v5 + # with: + # python-version: "3.13" + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install flake8 pytest + # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # - name: Lint with flake8 + # run: | + # # stop the build if there are Python syntax errors or undefined names + # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # - name: Test with pytest + # run: | + # pytest diff --git a/.gitignore b/.gitignore index 8d40064..98b7c0f 100644 --- a/.gitignore +++ b/.gitignore @@ -163,4 +163,6 @@ pyrightconfig.json # direnv files, used by load python venv .direnv/ -.envrc \ No newline at end of file +.envrc +.local.env +.envrc diff --git a/Makefile b/Makefile index 645a4c4..d61617d 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ run: lint: uvx ruff check . +fix: + uvx ruff check --fix . + test: lint uv run -m tests diff --git a/README.md b/README.md index 944fd70..ecf5ee6 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,7 @@ A `Makefile` is provided with the following targets: - `htmlcov` : run the unit tests and generate a full report in htmlcov/ Testing and coverage requires standing up a local testbed. For details, see [Design](docs/design.md). + + +## Adding `llm-redactor` branch +For the redactor feature. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 788d64a..a3f8cdd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,5 +7,7 @@ services: - DISCORD_TOKEN=${DISCORD_TOKEN} - REDMINE_TOKEN=${REDMINE_TOKEN} - REDMINE_URL=${REDMINE_URL} + volumes: + - ./redaction_queue.json:/app/redaction_queue.json #share queue file restart: on-failure:1 - network_mode: host + network_mode: host \ No newline at end of file diff --git a/data/custom_fields.json b/data/custom_fields.json index 127173f..26b8b41 100644 --- a/data/custom_fields.json +++ b/data/custom_fields.json @@ -1 +1,207 @@ -{"custom_fields":[{"id":2,"name":"Discord ID","description":"ID used to link user to their discord account, to enable integration","customized_type":"user","field_format":"string","regexp":"","min_length":null,"max_length":null,"is_required":false,"is_filter":false,"searchable":false,"multiple":false,"default_value":"","visible":true},{"id":4,"name":"syncdata","description":"Metadata used to sync the ticket with external sources.\r\n\r\nCurrent format is thread-id|zulu-timestamp.\r\n\r\nNot recommend to edit.","customized_type":"issue","field_format":"string","regexp":"","min_length":null,"max_length":null,"is_required":false,"is_filter":true,"searchable":false,"multiple":false,"default_value":"","visible":false,"trackers":[{"id":2,"name":"Infra-Field"},{"id":4,"name":"Software-Dev"},{"id":6,"name":"Infra-Config"},{"id":8,"name":"External-Comms-Intake"},{"id":9,"name":"Outreach-Partnerships"},{"id":10,"name":"Admin"},{"id":17,"name":"Research"},{"id":18,"name":"Mutual-Aid-Action"},{"id":19,"name":"SCN-Space"}],"roles":[{"id":3,"name":"Administrator"}]},{"id":5,"name":"To/CC","description":"Contains the To and Cc headers from the email that created the ticket.","customized_type":"issue","field_format":"string","regexp":"","min_length":null,"max_length":null,"is_required":false,"is_filter":false,"searchable":true,"multiple":false,"default_value":"","visible":true,"trackers":[{"id":2,"name":"Infra-Field"},{"id":4,"name":"Software-Dev"},{"id":6,"name":"Infra-Config"},{"id":8,"name":"External-Comms-Intake"},{"id":9,"name":"Outreach-Partnerships"},{"id":10,"name":"Admin"},{"id":13,"name":"Test-Reject"},{"id":17,"name":"Research"},{"id":18,"name":"Mutual-Aid-Action"},{"id":19,"name":"SCN-Space"}],"roles":[]}]} +{ + "custom_fields": [ + { + "id": 2, + "name": "Discord ID", + "description": "ID used to link user to their discord account, to enable integration", + "customized_type": "user", + "field_format": "string", + "regexp": "", + "min_length": null, + "max_length": null, + "is_required": false, + "is_filter": false, + "searchable": false, + "multiple": false, + "default_value": "", + "visible": true, + "editable": true + }, + { + "id": 4, + "name": "syncdata", + "description": "Metadata used to sync the ticket with external sources.\r\n\r\nCurrent format is thread-id|zulu-timestamp.\r\n\r\nNot recommend to edit.", + "customized_type": "issue", + "field_format": "string", + "regexp": "", + "min_length": null, + "max_length": null, + "is_required": false, + "is_filter": true, + "searchable": false, + "multiple": false, + "default_value": "", + "visible": false, + "editable": true, + "trackers": [ + { + "id": 2, + "name": "Infra-Field" + }, + { + "id": 4, + "name": "Software-Dev" + }, + { + "id": 6, + "name": "Infra-Config" + }, + { + "id": 8, + "name": "External-Comms-Intake" + }, + { + "id": 9, + "name": "Outreach-Partnerships" + }, + { + "id": 10, + "name": "Admin" + }, + { + "id": 17, + "name": "Research" + }, + { + "id": 18, + "name": "Mutual-Aid-Action" + }, + { + "id": 19, + "name": "SCN-Space" + } + ], + "roles": [ + { + "id": 3, + "name": "Administrator" + } + ] + }, + { + "id": 5, + "name": "To/CC", + "description": "Contains the To and Cc headers from the email that created the ticket.", + "customized_type": "issue", + "field_format": "string", + "regexp": "", + "min_length": null, + "max_length": null, + "is_required": false, + "is_filter": false, + "searchable": true, + "multiple": false, + "default_value": "", + "visible": true, + "editable": true, + "trackers": [ + { + "id": 2, + "name": "Infra-Field" + }, + { + "id": 4, + "name": "Software-Dev" + }, + { + "id": 6, + "name": "Infra-Config" + }, + { + "id": 8, + "name": "External-Comms-Intake" + }, + { + "id": 9, + "name": "Outreach-Partnerships" + }, + { + "id": 10, + "name": "Admin" + }, + { + "id": 13, + "name": "Test-Reject" + }, + { + "id": 17, + "name": "Research" + }, + { + "id": 18, + "name": "Mutual-Aid-Action" + }, + { + "id": 19, + "name": "SCN-Space" + } + ], + "roles": [] + }, + { + "id": 6, + "name": "redacted", + "description": "Keys and values for the redacted fields in the ticket.", + "customized_type": "issue", + "field_format": "string", + "regexp": "", + "min_length": null, + "max_length": null, + "is_required": false, + "is_filter": false, + "searchable": false, + "multiple": false, + "default_value": "", + "visible": false, + "editable": true, + "trackers": [ + { + "id": 2, + "name": "Infra-Field" + }, + { + "id": 4, + "name": "Software-Dev" + }, + { + "id": 6, + "name": "Infra-Config" + }, + { + "id": 8, + "name": "External-Comms-Intake" + }, + { + "id": 9, + "name": "Outreach-Partnerships" + }, + { + "id": 10, + "name": "Admin" + }, + { + "id": 13, + "name": "Test-Reject" + }, + { + "id": 17, + "name": "Research" + }, + { + "id": 18, + "name": "Mutual-Aid-Action" + }, + { + "id": 19, + "name": "SCN-Space" + } + ], + "roles": [ + { + "id": 3, + "name": "Administrator" + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/threader.md b/docs/threader.md index 5fef655..5ac5402 100644 --- a/docs/threader.md +++ b/docs/threader.md @@ -48,6 +48,14 @@ The `threader_job.sh` exists to make sure the `.env` file and the `venv` Python All output to stdout and stderr captured and logged to syslog with the tag "threader". +## Redactor Configuration + +If the `redactor` is being used, it should be configured in the `.env` file in the netbot deployment: + +``` +REDACTOR_URL=http://192.168.20.64:8000 +``` + ## Threader Logs: `/var/log/syslog` diff --git a/netbot/cog_scn.py b/netbot/cog_scn.py index 9c3e2a5..c3d675f 100644 --- a/netbot/cog_scn.py +++ b/netbot/cog_scn.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 """Cog to manage SCN-related functions""" + import logging import discord - -from discord.commands import option, SlashCommandGroup +from discord.commands import SlashCommandGroup, option from discord.ext import commands from discord.utils import basic_autocomplete -from redmine.model import Message, User -from redmine.redmine import Client, BLOCKED_TEAM_NAME - from netbot.netbot import NetBot, default_ticket +from redmine.model import Message, User +from redmine.redmine import BLOCKED_TEAM_NAME, Client log = logging.getLogger(__name__) @@ -24,12 +23,15 @@ # scn reindex + def setup(bot): bot.add_cog(SCNCog(bot)) log.info("initialized SCN cog") + class NewUserModal(discord.ui.Modal): """modal dialog to collect new user info""" + def __init__(self, redmine: Client, login: str, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.redmine = redmine @@ -38,7 +40,6 @@ def __init__(self, redmine: Client, login: str, *args, **kwargs) -> None: self.add_item(discord.ui.InputText(label="Last Name")) self.add_item(discord.ui.InputText(label="Email")) - async def callback(self, interaction: discord.Interaction): email = self.children[2].value first = self.children[0].value @@ -48,12 +49,18 @@ async def callback(self, interaction: discord.Interaction): user = self.redmine.user_mgr.register(self.login, email, first, last) if user is None: - log.error(f"Unable to create user for {self.login}, {first} {last}, {email}") - await interaction.response.send_message(f"Unable to create user for {self.login}") + log.error( + f"Unable to create user for {self.login}, {first} {last}, {email}" + ) + await interaction.response.send_message( + f"Unable to create user for {self.login}" + ) return # create the mapping so it the discord user can be found - self.redmine.user_mgr.create_discord_mapping(user, interaction.user.id, interaction.user.name) + self.redmine.user_mgr.create_discord_mapping( + user, interaction.user.id, interaction.user.name + ) log.debug(f"mapped discord new user: {interaction.user.name} -> {user.login}") embed = discord.Embed(title="Registered User") @@ -65,20 +72,19 @@ async def callback(self, interaction: discord.Interaction): class ApproveButton(discord.ui.Button): """Discord button to approve specific users""" + def __init__(self, bot_: discord.Bot, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.bot = bot_ - # TODO Move to user_mgr? - def find_registered_user(self, discord_name:str) -> User: + def find_registered_user(self, discord_name: str) -> User: """Search registered users for a matching discord ID""" for user in self.bot.redmine.user_mgr.get_registered(): if user.discord_id == discord_name: return user return None - async def callback(self, interaction: discord.Interaction): name = self.label @@ -86,13 +92,16 @@ async def callback(self, interaction: discord.Interaction): if user: self.bot.redmine.user_mgr.approve(user) # assign default groups? - await interaction.response.send_message(f"Approved registered user: @{name} {user.login} {user.name}") + await interaction.response.send_message( + f"Approved registered user: @{name} {user.login} {user.name}" + ) else: await interaction.response.send_message(f"User not found: {name}") class ApproveUserView(discord.ui.View): """Approve registered users with Discord controls""" + def __init__(self, bot_: discord.Bot, users: list[User]) -> None: self.bot = bot_ super().__init__() @@ -101,10 +110,12 @@ def __init__(self, bot_: discord.Bot, users: list[User]) -> None: for user in users: self.add_item(ApproveButton(self.bot, label=user.discord)) - - async def button_callback(self, button: discord.ui.Button, interaction: discord.Interaction): - await interaction.response.send_message(f"ApproveUserView: {button} {interaction}") - + async def button_callback( + self, button: discord.ui.Button, interaction: discord.Interaction + ): + await interaction.response.send_message( + f"ApproveUserView: {button} {interaction}" + ) async def callback(self, interaction: discord.Interaction): await interaction.response.send_message(f"ApproveUserView: {interaction}") @@ -113,6 +124,7 @@ async def callback(self, interaction: discord.Interaction): # FIXME Not yet implemented class IntakeView(discord.ui.View): """Perform intake""" + # to build, need: # - list of trackers # - list or priorities @@ -128,24 +140,30 @@ def __init__(self, bot_: discord.Bot) -> None: super().__init__() # Adds the dropdown to our View object - #self.add_item(PrioritySelect(self.bot)) - #self.add_item(discord.ui.InputText(label="Subject", row=1)) - #self.add_item(TrackerSelect(self.bot)) - + # self.add_item(PrioritySelect(self.bot)) + # self.add_item(discord.ui.InputText(label="Subject", row=1)) + # self.add_item(TrackerSelect(self.bot)) self.add_item(discord.ui.Button(label="Assign", row=4)) self.add_item(discord.ui.Button(label="Reject ticket subject", row=4)) self.add_item(discord.ui.Button(label="Block email@address.com", row=4)) - async def select_callback(self, select, interaction): # the function called when the user is done selecting options - await interaction.response.send_message(f"IntakeView.select_callback() selected: {select.values[0]}") + async def select_callback( + self, select, interaction + ): # the function called when the user is done selecting options + await interaction.response.send_message( + f"IntakeView.select_callback() selected: {select.values[0]}" + ) async def callback(self, interaction: discord.Interaction): - await interaction.response.send_message(f"IntakeView.allback() {interaction.data}") + await interaction.response.send_message( + f"IntakeView.allback() {interaction.data}" + ) class SCNCog(commands.Cog): """Cog to mange SCN-related functions""" + def __init__(self, bot): self.bot = bot self.formatter = bot.formatter @@ -153,84 +171,115 @@ def __init__(self, bot): # see https://github.com/Pycord-Development/pycord/blob/master/examples/app_commands/slash_cog_groups.py - scn = SlashCommandGroup("scn", "SCN admin commands") - + scn = SlashCommandGroup("scn", "SCN admin commands") def is_admin(self, user: discord.Member) -> bool: """Check if the given Discord memeber is in a authorized role""" # search user for "auth" role for role in user.roles: - if "auth" == role.name: ## FIXME + if "auth" == role.name: ## FIXME return True # auth role not found return False - # FIXME rename to "register"? @scn.command(description="Add a Discord user to redmine") - @option("ticket_id", description="ticket ID", autocomplete=basic_autocomplete(default_ticket)) - @option("member", description="Discord member collaborating with ticket", optional=True) - async def add(self, ctx:discord.ApplicationContext, redmine_login:str, member:discord.Member=None): + @option( + "ticket_id", + description="ticket ID", + autocomplete=basic_autocomplete(default_ticket), + ) + @option( + "member", description="Discord member collaborating with ticket", optional=True + ) + async def add( + self, + ctx: discord.ApplicationContext, + redmine_login: str, + member: discord.Member = None, + ): """add a Discord user to the Redmine ticketing integration""" - discord_id = ctx.user # by default, assume current user + discord_id = ctx.user # by default, assume current user if member: - log.info(f"Overriding current user={ctx.user.name} with member={member.name}") + log.info( + f"Overriding current user={ctx.user.name} with member={member.name}" + ) discord_id = member user = self.redmine.user_mgr.find(discord_id.name) if user: # check the id_from_user = user.discord_id - if id_from_user.id > 0: + if id_from_user and id_from_user.id > 0: # a valid - await ctx.respond(f"Discord user: {discord_id} is fully configured as redmine user: {user.login}") + await ctx.respond( + f"Discord user: {discord_id} is fully configured as redmine user: {user.login}" + ) else: # need to update - self.redmine.user_mgr.create_discord_mapping(user, discord_id.id, discord_id.name) - await ctx.respond(f"Discord user: {discord_id.id},{discord_id.name} has been paired with redmine user: {redmine_login}") + self.redmine.user_mgr.create_discord_mapping( + user, discord_id.id, discord_id.name + ) + await ctx.respond( + f"Discord user: {discord_id.id},{discord_id.name} has been paired with redmine user: {redmine_login}" + ) else: user = self.redmine.user_mgr.find(redmine_login) if user and self.is_admin(ctx.user): - self.redmine.user_mgr.create_discord_mapping(user, discord_id.id, discord_id.name) - await ctx.respond(f"Discord user: {discord_id.name} has been paired with redmine user: {redmine_login}") + self.redmine.user_mgr.create_discord_mapping( + user, discord_id.id, discord_id.name + ) + await ctx.respond( + f"Discord user: {discord_id.name} has been paired with redmine user: {redmine_login}" + ) else: # case: unknown redmine_login -> new user request: register new user - modal = NewUserModal(self.redmine, redmine_login, title="Register new user") + modal = NewUserModal( + self.redmine, redmine_login, title="Register new user" + ) await ctx.send_modal(modal) # reindex users after changes self.redmine.user_mgr.reindex_users() - @scn.command() - async def sync(self, ctx:discord.ApplicationContext): + async def sync(self, ctx: discord.ApplicationContext): """syncronize an existing ticket thread with redmine""" if isinstance(ctx.channel, discord.Thread): thread = ctx.channel ticket = await self.bot.sync_thread(thread) if ticket: - await ctx.respond(f"SYNC ticket {ticket.id} to thread: {thread.name} complete") + await ctx.respond( + f"SYNC ticket {ticket.id} to thread: {thread.name} complete" + ) else: # double-check thread name ticket_id = NetBot.parse_thread_title(thread.name) if ticket_id: - await ctx.respond(f"No ticket (#{ticket_id}) found for thread named: {thread.name}") + await ctx.respond( + f"No ticket (#{ticket_id}) found for thread named: {thread.name}" + ) else: # create new ticket subject = thread.name user = self.redmine.user_mgr.find(ctx.user.name) - message = Message(user.login, subject) # user.mail? - message.note = subject + "\n\nCreated by netbot by syncing Discord thread with same name." + message = Message(user.login, subject) # user.mail? + message.note = ( + subject + + "\n\nCreated by netbot by syncing Discord thread with same name." + ) ticket = self.redmine.ticket_mgr.create(user, message) # set tracker # TODO: search up all parents in hierarchy? - tracker = self.bot.redmine.ticket_mgr.get_tracker(thread.parent.name) + tracker = self.bot.redmine.ticket_mgr.get_tracker( + thread.parent.name + ) if tracker: log.debug(f"found {thread.parent.name} => {tracker}") params = { "tracker_id": str(tracker.id), - "notes": f"Setting tracker based on channel name: {thread.parent.name}" + "notes": f"Setting tracker based on channel name: {thread.parent.name}", } self.redmine.ticket_mgr.update(ticket.id, params, user.login) else: @@ -240,22 +289,20 @@ async def sync(self, ctx:discord.ApplicationContext): await thread.edit(name=f"Ticket #{ticket.id}: {ticket.subject}") # sync the thread - ticket = await self.bot.sync_thread(thread) # refesh the ticket + ticket = await self.bot.sync_thread(thread) # refesh the ticket await ctx.respond(self.bot.formatter.format_ticket(ticket)) - #OLD await ctx.respond(f"Cannot find ticket# in thread name: {ctx.channel.name}") # error + # OLD await ctx.respond(f"Cannot find ticket# in thread name: {ctx.channel.name}") # error else: - await ctx.respond("Not a thread.") # error - + await ctx.respond("Not a thread.") # error @scn.command() - async def reindex(self, ctx:discord.ApplicationContext): + async def reindex(self, ctx: discord.ApplicationContext): """reindex the all cached information""" self.redmine.reindex() - #self.bot.reindex() FIXME, once roles are working + # self.bot.reindex() FIXME, once roles are working await ctx.respond("Rebuilt redmine indices.") - # REMOVE - handled by discord roles # @scn.command(description="join the specified team") # async def join(self, ctx:discord.ApplicationContext, teamname:str , member: discord.Member=None): @@ -288,15 +335,15 @@ async def reindex(self, ctx:discord.ApplicationContext): # else: # await ctx.respond(f"Unknown Discord user: {discord_name}.") - - def find_role(self, ctx:discord.ApplicationContext, rolename:str) -> discord.Role | None: + def find_role( + self, ctx: discord.ApplicationContext, rolename: str + ) -> discord.Role | None: for role in ctx.guild.roles: if role.name == rolename: return role - @scn.command(description="list teams and members") - async def teams(self, ctx:discord.ApplicationContext, teamname:str=None): + async def teams(self, ctx: discord.ApplicationContext, teamname: str = None): # list teams, with members if teamname: team = self.find_role(ctx, teamname) @@ -304,8 +351,16 @@ async def teams(self, ctx:discord.ApplicationContext, teamname:str=None): await ctx.respond(self.formatter.format_team(ctx, team)) return else: - all_teams = ", ".join([team.name for team in ctx.guild.roles if team.name.endswith("-team")]) - await ctx.respond(f"Unknown team name: {teamname}\nTeams: {all_teams}") # error + all_teams = ", ".join( + [ + team.name + for team in ctx.guild.roles + if team.name.endswith("-team") + ] + ) + await ctx.respond( + f"Unknown team name: {teamname}\nTeams: {all_teams}" + ) # error else: # all teams buff = "" @@ -313,38 +368,38 @@ async def teams(self, ctx:discord.ApplicationContext, teamname:str=None): buff += self.formatter.format_team(ctx, team, inc_users=False) await ctx.respond(buff) - - #@scn.command() - #async def intake(self, ctx:discord.ApplicationContext): + # @scn.command() + # async def intake(self, ctx:discord.ApplicationContext): # """perform intake""" # # check team? admin?, provide reasonable error msg. # await ctx.respond("INTAKE #{ticket.id}", view=IntakeView(self.bot)) - @scn.command(description="list all open epics") - async def epics(self, ctx:discord.ApplicationContext): + async def epics(self, ctx: discord.ApplicationContext): """List all the epics, grouped by tracker""" # get the epics. epics = self.redmine.ticket_mgr.get_epics() # format the epics and respond await ctx.respond(embeds=self.bot.formatter.epics_embed(ctx, epics)) - @scn.command(description="list blocked email") - async def blocked(self, ctx:discord.ApplicationContext): + async def blocked(self, ctx: discord.ApplicationContext): team = self.redmine.user_mgr.cache.get_team_by_name(BLOCKED_TEAM_NAME) if team: await ctx.respond(self.formatter.format_team(team)) else: - await ctx.respond(f"Expected team {BLOCKED_TEAM_NAME} not configured") # error - + await ctx.respond( + f"Expected team {BLOCKED_TEAM_NAME} not configured" + ) # error # ticket 484 - http://10.10.0.218/issues/484 # block users based on name (not discord membership) - @scn.command(description="block specific a email address and reject all related tickets") - async def block(self, ctx:discord.ApplicationContext, username:str): + @scn.command( + description="block specific a email address and reject all related tickets" + ) + async def block(self, ctx: discord.ApplicationContext, username: str): log.debug(f"blocking {username}") - #user = self.redmine.lookup_user(username) + # user = self.redmine.lookup_user(username) user = self.redmine.user_mgr.find(username) if user: # add the user to the blocked list @@ -352,14 +407,15 @@ async def block(self, ctx:discord.ApplicationContext, username:str): # search and reject all tickets from that user for ticket in self.redmine.ticket_mgr.get_by(user): self.redmine.ticket_mgr.reject_ticket(ticket.id) - await ctx.respond(f"Blocked user: {user.login} and rejected all created tickets") + await ctx.respond( + f"Blocked user: {user.login} and rejected all created tickets" + ) else: log.debug("trying to block unknown user '{username}', ignoring") await ctx.respond(f"Unknown user: {username}") - @scn.command(description="unblock specific a email address") - async def unblock(self, ctx:discord.ApplicationContext, username:str): + async def unblock(self, ctx: discord.ApplicationContext, username: str): log.debug(f"Unblocking {username}") user = self.redmine.user_mgr.find(username) if user: @@ -369,28 +425,60 @@ async def unblock(self, ctx:discord.ApplicationContext, username:str): log.debug("trying to unblock unknown user '{username}', ignoring") await ctx.respond(f"Unknown user: {username}") - @scn.command(name="force-notify", description="Force ticket notifications") - async def force_remind(self, ctx:discord.ApplicationContext): + async def force_remind(self, ctx: discord.ApplicationContext): await ctx.respond("Sending reminders for dusty tickets....") await self.bot.remind_dusty_tickets() - @scn.command(name="force-recycle", description="Force ticket notifications") - async def force_recycle(self, ctx:discord.ApplicationContext): + async def force_recycle(self, ctx: discord.ApplicationContext): await ctx.respond("Recycling dusty, old tickets....") await self.bot.recycle_tickets() - @scn.command(description="List and approve registered new users") - async def approve(self, ctx:discord.ApplicationContext): + async def approve(self, ctx: discord.ApplicationContext): if self.is_admin(ctx.user): # get the registered users users = self.bot.redmine.user_mgr.get_registered() if len(users) > 0: - await ctx.respond("Approve Registered Users", view=ApproveUserView(self.bot, users)) + await ctx.respond( + "Approve Registered Users", view=ApproveUserView(self.bot, users) + ) else: await ctx.respond("No pending registered users.") else: await ctx.respond("Must be authorized admin to approve Redmine users.") + + # -------- + + # @scn.command(description="Redact a ticket") + # async def redact(self, ctx:discord.ApplicationContext, ticket_id: int): + # if self.is_admin(ctx.user): + # ticket = self.redmine.ticket_mgr.get(ticket_id) + # if ticket: + # ticket = self.redmine.redact_ticket(ticket) + # await self.bot.formatter.print_ticket(ticket, ctx) + # else: + # # TODO user error + # await ctx.respond(f"Ticket {ticket_id} not found.", ephemeral=True) + + # @scn.command(description="Display ticket details with redacted info displayed") + # async def unredact(self, ctx:discord.ApplicationContext, ticket_id: int): + # if self.bot.is_pii_admin(ctx.user): #TODO CHANGE LATER + # await ctx.defer(ephemeral=True) # TODO CHANGE LATER + # ticket = self.redmine.ticket_mgr.get(ticket_id) + # if ticket: + # # ticket = self.redmine.unredact_ticket(ticket) TODO CHANGE LATER + # #await self.bot.formatter.print_ticket(ticket, ctx) + + # # await ctx.respond(embed=self.formatter.ticket_embed(ctx, ticket), ephemeral=True) + # await ctx.followup.send(embed=self.formatter.ticket_embed(ctx, ticket), ephemeral=True) # Changed to followup TODO CHANGE LATER + + # else: + # # TODO user error + # # await ctx.respond(f"Ticket {ticket_id} not found.") + # await ctx.followup.send(f"Ticket {ticket_id} not found.") # Changed to followup TODO CHANGE LATER + # else + # ticket = self.redmine.ticket_mgr.get(ticket_id) + # await ctx.followup.send(embed=self.formatter.ticket_embed(ctx, ticket), ephemeral=True) diff --git a/netbot/cog_tickets.py b/netbot/cog_tickets.py index bc23583..dfeba74 100644 --- a/netbot/cog_tickets.py +++ b/netbot/cog_tickets.py @@ -253,34 +253,83 @@ async def callback(self, interaction: discord.Interaction): await interaction.response.send_message(embeds=[embed]) +# class EditDescriptionModal(discord.ui.Modal): +# """modal dialog to edit the ticket subject and description""" +# def __init__(self, redmine: Client, ticket: Ticket, *args, **kwargs) -> None: +# super().__init__(*args, **kwargs) +# # Note: redmine must be available in callback, as the bot is not +# # available thru the Interaction. +# self.redmine = redmine +# self.ticket_id = ticket.id +# self.add_item(discord.ui.InputText(label="Description", +# value=ticket.description, +# style=InputTextStyle.paragraph)) + + +# async def callback(self, interaction: discord.Interaction): +# description = self.children[0].value +# log.debug(f"callback: {description}") + +# user = self.redmine.user_mgr.find_discord_user(interaction.user.name) + +# fields = { +# "description": description, +# } +# ticket = self.redmine.ticket_mgr.update(self.ticket_id, fields, user.login) + +# embed = discord.Embed(title=f"Updated ticket {ticket.id} description") +# embed.add_field(name="Description", value=ticket.description) + +# await interaction.response.send_message(embeds=[embed]) + class EditDescriptionModal(discord.ui.Modal): """modal dialog to edit the ticket subject and description""" - def __init__(self, redmine: Client, ticket: Ticket, *args, **kwargs) -> None: + def __init__(self, redmine: Client, ticket: Ticket, bot: NetBot, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # Note: redmine must be available in callback, as the bot is not - # available thru the Interaction. self.redmine = redmine self.ticket_id = ticket.id - self.add_item(discord.ui.InputText(label="Description", - value=ticket.description, - style=InputTextStyle.paragraph)) - + self.bot = bot + self.add_item(discord.ui.InputText( + label="Description", + value=ticket.get_custom_field("unredacted") or ticket.description, + style=InputTextStyle.paragraph + )) async def callback(self, interaction: discord.Interaction): + from redaction_queue import RedactionQueue + description = self.children[0].value - log.debug(f"callback: {description}") - + log.debug(f"Edit callback for ticket #{self.ticket_id}") + + queue = RedactionQueue() + + # Check if ticket is locked + # if queue.is_locked(self.ticket_id): + # await interaction.response.send_message( + # "Can't edit right now. Please wait and try again in a few minutes.", + # ephemeral=True + # ) + # return + + # Get user info user = self.redmine.user_mgr.find_discord_user(interaction.user.name) - - fields = { - "description": description, + user_info = { + "name": user.name if user else interaction.user.name, + "login": user.login if user else None, + "discord_id": interaction.user.id } - ticket = self.redmine.ticket_mgr.update(self.ticket_id, fields, user.login) - - embed = discord.Embed(title=f"Updated ticket {ticket.id} description") - embed.add_field(name="Description", value=ticket.description) - - await interaction.response.send_message(embeds=[embed]) + + # Add to queue + job_id = queue.add_edit_job(self.ticket_id, description, user_info) + + await interaction.response.send_message( + f"Your edit has been queued for redaction.\n" + f"This will take approximately 15-20 minutes.\n\n" + f"Ticket #{self.ticket_id} is locked during this process.", + ephemeral=True + ) + + log.info(f"Queued edit job {job_id} for ticket #{self.ticket_id}") # distinct from above. takes app-context @@ -302,6 +351,10 @@ def __init__(self, bot:NetBot): self.bot:NetBot = bot self.redmine: Client = bot.redmine + def is_pii_admin(self, user: discord.Member) -> bool: + """Check if user has PII admin role""" + return self.bot.is_pii_admin(user) + # see https://github.com/Pycord-Development/pycord/blob/master/examples/app_commands/slash_cog_groups.py ticket = SlashCommandGroup("ticket", "ticket commands") @@ -382,18 +435,28 @@ async def query(self, ctx: discord.ApplicationContext, term:str = ""): else: await ctx.respond(f"Zero results for: `{term}`") - - @ticket.command(description="Get ticket details") @option("ticket_id", description="ticket ID", autocomplete=basic_autocomplete(default_ticket)) - async def details(self, ctx: discord.ApplicationContext, ticket_id:int): - """Update status on a ticket, using: unassign, resolve, progress""" - #log.debug(f"found user mapping for {ctx.user.name}: {user}") + async def details(self, ctx: discord.ApplicationContext, ticket_id: int): + #Show ticket details always ephemeral + #PII admin: pulls unredacted CF and swaps into description for display + #Regular user: shows description as is (already redacted) ticket = self.redmine.ticket_mgr.get(ticket_id, include="children,watchers") - if ticket: - await self.bot.formatter.print_ticket(ticket, ctx) - else: - await ctx.respond(f"Ticket {ticket_id} not found.") # print error + + if not ticket: + await ctx.respond(f"Ticket {ticket_id} not found.", ephemeral=True) + return + + if self.is_pii_admin(ctx.user): + #pull original PII from unredacted CF and swap into description for UI DISPLAY ONLY + unredacted_value = ticket.get_custom_field("unredacted") + if unredacted_value: + ticket.description = unredacted_value + + await ctx.respond( + embed=self.bot.formatter.ticket_embed(ctx, ticket), + ephemeral=True + ) @ticket.command(description="Collaborate on a ticket") @@ -523,6 +586,7 @@ async def assign(self, ctx: discord.ApplicationContext, ticket_id:int, member:di async def create_thread(self, ticket:Ticket, ctx:discord.ApplicationContext): log.info(f"creating a new thread for ticket #{ticket.id} in channel: {ctx.channel.name}") thread_name = f"Ticket #{ticket.id}: {ticket.subject}" + if isinstance(ctx.channel, discord.Thread): log.debug(f"creating thread in parent channel {ctx.channel.parent.name}, for {ticket}") thread = await ctx.channel.parent.create_thread(name=thread_name, type=discord.ChannelType.public_thread) @@ -533,7 +597,6 @@ async def create_thread(self, ticket:Ticket, ctx:discord.ApplicationContext): await thread.send(self.bot.formatter.format_ticket_details(ticket)) return thread - @ticket.command(name="new", description="Create a new ticket") @option("title", description="Title of the new SCN ticket") async def create_new_ticket(self, ctx: discord.ApplicationContext, title:str): @@ -806,16 +869,56 @@ async def due(self, ctx: discord.ApplicationContext, date:str): await ctx.respond("Command only valid in ticket thread. No ticket info found in this thread.") + # @ticket.command(name="description", description="Edit the description of a ticket") + # async def edit_description(self, ctx: discord.ApplicationContext): + # # pop the the edit description embed + # ticket_id = NetBot.parse_thread_title(ctx.channel.name) + # ticket = self.redmine.ticket_mgr.get(ticket_id) + # if ticket: + # modal = EditDescriptionModal(self.redmine, ticket, title=f"Editing ticket #{ticket.id}") + # await ctx.send_modal(modal) + # else: + # await ctx.respond(f"Cannot find ticket for {ctx.channel}") + @ticket.command(name="description", description="Edit the description of a ticket") async def edit_description(self, ctx: discord.ApplicationContext): - # pop the the edit description embed + from redaction_queue import RedactionQueue + + # Check PII admin permission + if not self.is_pii_admin(ctx.user): + await ctx.respond( + "You don't have permission to edit ticket descriptions.", + ephemeral=True + ) + return + + # Get ticket from thread name ticket_id = NetBot.parse_thread_title(ctx.channel.name) + if not ticket_id: + await ctx.respond( + "This command only works in ticket threads.", + ephemeral=True + ) + return + + # Check if ticket is locked + queue = RedactionQueue() + if queue.is_locked(ticket_id): + await ctx.respond( + "Ticket is currently being redacted. Please wait and try again in a few moments.", + ephemeral=True + ) + return + + # Get ticket ticket = self.redmine.ticket_mgr.get(ticket_id) - if ticket: - modal = EditDescriptionModal(self.redmine, ticket, title=f"Editing ticket #{ticket.id}") - await ctx.send_modal(modal) - else: - await ctx.respond(f"Cannot find ticket for {ctx.channel}") + if not ticket: + await ctx.respond(f"Cannot find ticket #{ticket_id}", ephemeral=True) + return + + # Show edit modal + modal = EditDescriptionModal(self.redmine, ticket, self.bot, title=f"Editing ticket #{ticket.id}") + await ctx.send_modal(modal) @ticket.command(name="parent", description="Set a parent ticket for ") diff --git a/netbot/formatting.py b/netbot/formatting.py index 4f8dc17..66021de 100644 --- a/netbot/formatting.py +++ b/netbot/formatting.py @@ -268,6 +268,7 @@ def format_ticket_details(self, ticket:Ticket) -> str: # ### Description # description text #link_padding = ' ' * (5 - len(str(ticket.id))) # field width = 6 + status = self.format_icon(ticket.status) priority = self.format_icon(ticket.priority) created_age = synctime.age_str(ticket.created_on) @@ -281,9 +282,17 @@ def format_ticket_details(self, ticket:Ticket) -> str: details += f"**Priority:** {priority}\n" details += f"**Assignee:** {assigned}\n" details += f"**Category:** {ticket.category}\n" - if ticket.to or ticket.cc: - details += f"**To:** {', '.join(ticket.to)} **Cc:** {', '.join(ticket.cc)}\n" + # if ticket.to or ticket.cc: + # details += f"**To:** {', '.join(ticket.to)} **Cc:** {', '.join(ticket.cc)}\n" + #removed since reveals PII in ticket thread + + # #TODO CHANGE LATER + # redacted_desc = ticket.get_custom_field("redacted") + # description = redacted_desc if redacted_desc else ticket.description + # #TODO CHANGE LATER + # details += f"### Description\n{description}" + # Description is always public-safe (redacted text stored directly in description) details += f"### Description\n{ticket.description}" return details @@ -349,7 +358,7 @@ def lookup_discord_user(self, ctx: discord.ApplicationContext, name:str) -> disc def get_user_id(self, ctx: discord.ApplicationContext, ticket:Ticket) -> str: if ticket is None or ticket.assigned_to is None: - return "" + return None user_str = self.format_discord_member(ctx, ticket.assigned_to.id) if not user_str: @@ -359,12 +368,12 @@ def get_user_id(self, ctx: discord.ApplicationContext, ticket:Ticket) -> str: def format_discord_member(self, ctx: discord.ApplicationContext, user_id:int) -> str: - user = ctx.bot.redmine.user_mgr.get(user_id) # call to cache + user = ctx.bot.redmine.user_mgr.cache.get(user_id) # call to cache directly if user and user.discord_id: return f"<@!{user.discord_id.id}>" if user: return user.name - return "" + return None def format_collaborators(self, ctx: discord.ApplicationContext, ticket:Ticket) -> str: diff --git a/netbot/netbot.py b/netbot/netbot.py index a3a2b68..4ce08c2 100755 --- a/netbot/netbot.py +++ b/netbot/netbot.py @@ -610,6 +610,14 @@ def is_admin(self, user: discord.Member) -> bool: # auth role not found return False + def is_pii_admin(self, user: discord.Member) -> bool: + """Check if user has PII admin role (can view unredacted content and edit tickets)""" + # search user for "pii_admin" role + for role in user.roles: + if "pii_admin" == role.name: + return True + # pii_admin role not found + return False def main(): """netbot main function""" diff --git a/redaction_queue.json b/redaction_queue.json new file mode 100644 index 0000000..f68e630 --- /dev/null +++ b/redaction_queue.json @@ -0,0 +1,131 @@ +{ + "queue": [ + { + "id": "edit-3015-1771907301", + "type": "edit", + "ticket_id": 3015, + "description": "Hi team,\n\nI spoke with Daniel Kim about the WiFi issue at the Rainier site. He said the router has been intermittently dropping connection since yesterday afternoon. You can reach him at daniel.kim@gmail.com or 206-555-2187 if you need details.\n\nThe equipment is located at 3124 S Alaska St, Seattle, WA 98108. Please let me know once someone is able to take a look.\n\nThanks,\nJay Robert", + "user": { + "name": "Robert Terracin", + "login": "iamlamp", + "discord_id": 469362479059959819 + }, + "timestamp": "2026-02-24T04:28:21.329895", + "status": "failed", + "error": "API returned status 500: {\"detail\":\"Failed to parse model output: Failed to parse model output: Invalid control character at: line 2 column 33 (char 34)\"}" + }, + { + "id": "edit-3017-1771968205", + "type": "edit", + "ticket_id": 3017, + "description": "Hi team,\n\nI spoke with Daniel Kim about the WiFi issue at the Rainier site. He said the router has been intermittently dropping connection since yesterday afternoon. You can reach him at daniel.kim@gmail.com or 206-555-2187 if you need more details.\n\n127.0.0.1\n\nThe equipment is located at 3124 S Alaska St, Seattle, WA 98108. Please let me know once someone is able to take a look.\n\nThanks,\nJay Robert", + "user": { + "name": "Dan", + "login": "danbSCN", + "discord_id": 739669349199118426 + }, + "timestamp": "2026-02-24T21:23:25.370608", + "status": "failed", + "error": "LLM API busy after max retries" + }, + { + "id": "edit-3017-1771968664", + "type": "edit", + "ticket_id": 3017, + "description": "Hi team,\n\nI spoke with Daniel Kim about the WiFi issue at the Rainier site. He said the router has been intermittently dropping connection since yesterday afternoon. You can reach him at daniel.kim@gmail.com or 206-555-2187 if you need more details.\n\nIP address: 172.0.0.1\n\nThe equipment is located at 3124 S Alaska St, Seattle, WA 98108. Please let me know once someone is able to take a look.\n\nThanks,\nJay Robert", + "user": { + "name": "Rudra Singh", + "login": "rudra", + "discord_id": 1117682047369089115 + }, + "timestamp": "2026-02-24T21:31:04.007547", + "status": "failed", + "error": "LLM API busy after max retries" + }, + { + "id": "edit-3182-1773277492", + "type": "edit", + "ticket_id": 3182, + "description": "(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted)\n\nHi team,\nI\u2019m having trouble accessing the shared drive this morning. Every time I\ntry to open \\\\acme-fs01\\Projects, I get an error that says \u201cNetwork path\nnot found.\u201d I\u2019ve tried restarting my laptop (ACMELAP123-BSM) and verified\nI\u2019m connected to VPN, but the issue persists.\n\nCan someone take a look? I\u2019ve got a report due this afternoon that depends\non those files.\n\nYou can expense new hardware for me. My credit card number is\n1234-1111-2222-3334, exp 10/26, security code 999.\n\nThanks,\n\nBob Smith\nSenior Data Analyst\nACME Corporation\nWork phone: (206) 555-1389\nEmployee ID: 104382\nSent from my ACME-issued laptop\n(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted).", + "user": { + "name": "Rudra Singh", + "login": "rudra", + "discord_id": 1117682047369089115 + }, + "timestamp": "2026-03-12T01:04:52.235124", + "status": "failed", + "error": "API returned status 500: {\"detail\":\"Failed to parse model output: Failed to parse model output: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)\"}" + }, + { + "id": "edit-3182-1773294724", + "type": "edit", + "ticket_id": 3182, + "description": "(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted)\n\nHi team,\nI\u2019m having trouble accessing the shared drive this morning. Every time I\ntry to open \\\\acme-fs01\\Projects, I get an error that says \u201cNetwork path\nnot found.\u201d I\u2019ve tried restarting my laptop (ACMELAP123-BSM) and verified\nI\u2019m connected to VPN, but the issue persists.\n\nCan someone take a look? I\u2019ve got a report due this afternoon that depends\non those files.\n\nYou can expense new hardware for me. My credit card number is\n1234-1111-2222-3334, exp 10/26, security code 999.\n\nThanks,\n\nBob Smith\nSenior Data Analyst\nACME Corporation\nWork phone: (206) 555-1389\nEmployee ID: 104382\nSent from my ACME-issued laptop\n(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted).", + "user": { + "name": "Rudra Singh", + "login": "rudra", + "discord_id": 1117682047369089115 + }, + "timestamp": "2026-03-12T05:52:04.082416", + "status": "failed", + "error": "API returned status 500: {\"detail\":\"Failed to parse model output: Failed to parse model output: Invalid \\\\escape: line 2 column 119 (char 120)\"}" + }, + { + "id": "edit-3182-1773295877", + "type": "edit", + "ticket_id": 3182, + "description": "(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted)\n\nHi team,\nI\u2019m having trouble accessing the shared drive this morning. Every time I\ntry to open \\\\acme-fs01\\Projects, I get an error that says \u201cNetwork path\nnot found.\u201d I\u2019ve tried restarting my laptop (ACMELAP123-BSM) and verified\nI\u2019m connected to VPN, but the issue persists.\n\nCan someone take a look? I\u2019ve got a report due this afternoon that depends\non those files.\n\nYou can expense new hardware for me. My credit card number is\n1234-1111-2222-3334, exp 10/26, security code 999.\n\nThanks,\n\nBob Smith\nSenior Data Analyst\nACME Corporation\nWork phone: (206) 555-1389\nEmployee ID: 104382\nSent from my ACME-issued laptop\n(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted).", + "user": { + "name": "Rudra Singh", + "login": "rudra", + "discord_id": 1117682047369089115 + }, + "timestamp": "2026-03-12T06:11:17.375759", + "status": "failed", + "error": "API returned status 500: {\"detail\":\"Failed to parse model output: Failed to parse model output: Invalid \\\\escape: line 2 column 119 (char 120)\"}" + }, + { + "id": "edit-3182-1773297224", + "type": "edit", + "ticket_id": 3182, + "description": "(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted)\n\nHi team,\nI\u2019m having trouble accessing the shared drive this morning. Every time I\ntry to open \\\\acme-fs01\\Projects, I get an error that says \u201cNetwork path\nnot found.\u201d I\u2019ve tried restarting my laptop (ACMELAP123-BSM) and verified\nI\u2019m connected to VPN, but the issue persists.\n\nCan someone take a look? I\u2019ve got a report due this afternoon that depends\non those files.\n\nYou can expense new hardware for me. My credit card number is\n1234-1111-2222-3334, exp 10/26, security code 999.\n\nThanks,\n\nBob Smith\nSenior Data Analyst\nACME Corporation\nWork phone: (206) 555-1389\nEmployee ID: 104382\nSent from my ACME-issued laptop\n(note: this is a test email by Tom for purposes of testing the discord\nredaction feature. Information here is not real and this ticket can be\ndeleted).", + "user": { + "name": "Rudra Singh", + "login": "rudra", + "discord_id": 1117682047369089115 + }, + "timestamp": "2026-03-12T06:33:44.729389", + "status": "failed", + "error": "API returned status 500: {\"detail\":\"Failed to parse model output: Failed to parse model output: Invalid \\\\escape: line 2 column 121 (char 122)\"}" + }, + { + "id": "edit-3091-1773844658", + "type": "edit", + "ticket_id": 3091, + "description": "TODO:\n- Run Esther thro latest changes\n- Clarify how to handle wg3 \n- handle existing/old IPs correction\n- implement error-catching logic\n- Test on \"full\" vyos output\n- Pipe vyos output in, rather than a txt/json?\n- Test on actual SCN netbox", + "user": { + "name": "Gabrielle Strandquist", + "login": "ellie", + "discord_id": 1096539392757215314 + }, + "timestamp": "2026-03-18T14:37:38.798737", + "status": "failed", + "error": "API returned status 500: {\"detail\":\"Failed to parse model output: Failed to parse model output: Expecting ',' delimiter: line 2 column 200 (char 201)\"}" + }, + { + "id": "edit-3090-1774985812", + "type": "edit", + "ticket_id": 3090, + "description": "Created by Discord user dan.con in channel outreach. \nPlanning to acquire merch and info to promote SCN at the annual Hamvention in Dayton, OH. Event draws in ham radio operators throughout the country. https://greenecoexpocenter.com/events/hamvention.\n\nAlso created printable stl file to create SCN coins to share in tandem with merch.", + "user": { + "name": "Dan", + "login": "danbSCN", + "discord_id": 739669349199118426 + }, + "timestamp": "2026-03-31T19:36:52.071589", + "status": "failed", + "error": "LLM API busy after max retries" + } + ], + "locked_tickets": {} +} diff --git a/redaction_queue.py b/redaction_queue.py new file mode 100644 index 0000000..dbe7e26 --- /dev/null +++ b/redaction_queue.py @@ -0,0 +1,151 @@ +import json +import logging +from pathlib import Path +from typing import Optional, Dict +from datetime import datetime +from threading import Lock + +log = logging.getLogger(__name__) + +QUEUE_FILE = Path("/home/scn/netbot-redacted/redaction_queue.json") + + +class RedactionQueue: + """Manages redaction jobs from both IMAP and Discord edits""" + + def __init__(self): + if Path("/app").exists(): + # Running in Docker container (netbot) + self.queue_file = Path("/app/redaction_queue.json") + else: + # Running on host (threader-daemon) + self.queue_file = Path("/home/scn/netbot-redacted/redaction_queue.json") + self.lock = Lock() + self._ensure_queue_file() + + def _ensure_queue_file(self): + """Create queue file if it doesn't exist""" + if not self.queue_file.exists(): + self._save_state({"queue": [], "locked_tickets": {}}) + + def _load_state(self) -> Dict: + """Load queue state from disk""" + try: + with open(self.queue_file, 'r') as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + log.error(f"Error loading queue: {e}, resetting") + return {"queue": [], "locked_tickets": {}} + + def _save_state(self, state: Dict): + """Save queue state to disk""" + with open(self.queue_file, 'w') as f: + json.dump(state, f, indent=2) + + def add_edit_job(self, ticket_id: int, description: str, user_info: dict) -> str: + """ + Add ticket edit to queue + Returns: job_id + """ + with self.lock: + state = self._load_state() + + job_id = f"edit-{ticket_id}-{int(datetime.now().timestamp())}" + job = { + "id": job_id, + "type": "edit", + "ticket_id": ticket_id, + "description": description, + "user": user_info, + "timestamp": datetime.now().isoformat(), + "status": "pending" + } + + state["queue"].append(job) + self._save_state(state) + + log.info(f"Added edit job to queue: {job_id}") + return job_id + + def get_next_job(self) -> Optional[Dict]: + """Get next pending job (FIFO)""" + with self.lock: + state = self._load_state() + log.info(f"Checking queue: {len(state['queue'])} total jobs") + + for job in state["queue"]: + log.info(f"Job {job['id']}: status={job.get('status', 'NO STATUS')}") + if job["status"] == "pending": + log.info(f"Found pending job: {job['id']}") + return job + + log.info("No pending jobs found") + return None + + def mark_processing(self, job_id: str): + """Mark job as processing""" + with self.lock: + state = self._load_state() + + for job in state["queue"]: + if job["id"] == job_id: + job["status"] = "processing" + break + + self._save_state(state) + + def mark_complete(self, job_id: str): + """Mark job as complete and remove from queue""" + with self.lock: + state = self._load_state() + + state["queue"] = [j for j in state["queue"] if j["id"] != job_id] + self._save_state(state) + + log.info(f"Completed job: {job_id}") + + def mark_failed(self, job_id: str, error: str): + """Mark job as failed""" + with self.lock: + state = self._load_state() + + for job in state["queue"]: + if job["id"] == job_id: + job["status"] = "failed" + job["error"] = error + break + + self._save_state(state) + + def lock_ticket(self, ticket_id: int, job_type: str = "edit"): + """Lock ticket during redaction""" + with self.lock: + state = self._load_state() + + state["locked_tickets"][str(ticket_id)] = { + "locked_at": datetime.now().isoformat(), + "type": job_type + } + + self._save_state(state) + log.info(f"Locked ticket #{ticket_id}") + + def unlock_ticket(self, ticket_id: int): + """Unlock ticket after completion""" + with self.lock: + state = self._load_state() + + if str(ticket_id) in state["locked_tickets"]: + del state["locked_tickets"][str(ticket_id)] + self._save_state(state) + log.info(f"Unlocked ticket #{ticket_id}") + + def is_locked(self, ticket_id: int) -> bool: + """Check if ticket is currently being redacted""" + state = self._load_state() + return str(ticket_id) in state["locked_tickets"] + + def has_pending_jobs(self) -> bool: + """Check if there are any pending jobs""" + state = self._load_state() + return any(j["status"] == "pending" for j in state["queue"]) \ No newline at end of file diff --git a/redactor/__init__.py b/redactor/__init__.py new file mode 100644 index 0000000..a1d5447 --- /dev/null +++ b/redactor/__init__.py @@ -0,0 +1 @@ +"""tests module""" diff --git a/redactor/redactor_client.py b/redactor/redactor_client.py new file mode 100644 index 0000000..c17f3fa --- /dev/null +++ b/redactor/redactor_client.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +HTTP client for calling the remote LLM redaction API +Used by threader on redmine2 to call llm server +""" + +import logging +import requests +import time + +log = logging.getLogger(__name__) + +# API configuration +MAX_RETRIES = 3 +RETRY_DELAY = 30 # seconds +TIMEOUT = 5 # seconds + +class RedactedText: + """Same interface as local redactor""" + def __init__(self, text: str, fields: dict): + self.text = text + self.fields = fields + + def __str__(self): + return self.text + + def unredact(self) -> str: + """Restore original PII""" + import re + pattern = re.compile(r"\[(\w+)\]") + restored_text = self.text + + for match in pattern.finditer(self.text): + placeholder_with_brackets = match.group(0) # [LastName1] + key = match.group(1) # LastName1 + + if key not in self.fields: + log.error(f"Expected field, {key}, not provided.") + continue + + value = self.fields[key] + restored_text = restored_text.replace(placeholder_with_brackets, value) + + return restored_text + + +class RedactorClient: + """HTTP client for remote redaction API""" + + def __init__(self, + api_url: str, + num_retrys: int = MAX_RETRIES, + retry_delay: int = RETRY_DELAY, + timeout: int = TIMEOUT): + self.api_url = api_url + self.retries = num_retrys + self.retry_delay = retry_delay + self.session = requests.Session() + self.session.headers.update({"Content-Type": "application/json"}) + + # Test connection + try: + response = self.session.get(f"{self.api_url}/health", timeout=timeout) + if response.status_code == 200: + log.info(f"Connected to LLM API at {self.api_url}") + else: + log.warning(f"LLM API health check failed: {response.status_code}") + except requests.exceptions.RequestException as e: + log.error(f"Failed to connect to LLM API at {self.api_url}: {e}") + raise RuntimeError(f"LLM API unavailable: {e}") + + def redact_text(self, text: str) -> RedactedText: + """ + Redact PII from text using remote API + + Args: + text: Original text to redact + + Returns: + RedactedText object with redacted text and PII mapping + + Raises: + RuntimeError: If API call fails after retries + """ + if not text or not text.strip(): + return RedactedText("", {}) + + for attempt in range(1, self.retries + 1): + try: + log.info(f"Calling LLM API (attempt {attempt}/{self.retries})...") + + response = self.session.post( + f"{self.api_url}/redact", + json={"text": text}, + timeout=1800 # 30 minutes (was 1200) # NOTE: This is a big timeout. + ) + + if response.status_code == 200: + data = response.json() + log.info(f"Redaction complete ({data['processing_time']:.1f}s)") + + return RedactedText( + text=data["redacted_text"], + fields=data["properties_redacted"] + ) + + elif response.status_code == 503: + # Server busy, retry + log.warning(f"LLM API busy, retrying in {self.re}s...") + if attempt < self.retries: + time.sleep(self.retry_delay) + continue + else: + raise RuntimeError("LLM API busy after max retries") + + else: + raise RuntimeError(f"API returned status {response.status_code}: {response.text}") + + except requests.exceptions.Timeout: + log.error(f"API timeout on attempt {attempt}") + if attempt < self.retries: + log.info(f"Retrying in {self.retry_delay}s...") + time.sleep(self.retry_delay) + else: + raise RuntimeError("API timeout after max retries") + + except requests.exceptions.RequestException as e: + log.error(f"API request failed: {e}") + if attempt < self.retries: + log.info(f"Retrying in {self.retry_delay}s...") + time.sleep(self.retry_delay) + else: + raise RuntimeError(f"API request failed after max retries: {e}") + + raise RuntimeError("Redaction failed after all retries") + + +# # For compatibility with existing code +# class Redactor(RedactorClient): +# """Alias for backward compatibility""" +# pass + + +# if __name__ == "__main__": +# # Test the client +# logging.basicConfig(level=logging.INFO) + +# client = RedactorClient() + +# test_text = "Contact John Smith at john.smith@example.com or call 555-123-4567" +# result = client.redact_text(test_text) + +# print(f"Original: {test_text}") +# print(f"Redacted: {result.text}") +# print(f"Fields: {result.fields}") +# print(f"Unredacted: {result.unredact()}") diff --git a/redmine/model.py b/redmine/model.py index bd5c7c1..b568086 100644 --- a/redmine/model.py +++ b/redmine/model.py @@ -334,6 +334,7 @@ class ParentTicket: SYNC_FIELD_NAME = "syncdata" TO_CC_FIELD_NAME = "To/CC" +REDACTOR_FIELD_NAME = "unredacted" @dataclass @@ -484,6 +485,20 @@ def age_str(self) -> str: return synctime.age_str(self.updated_on) + @property + def redacted_fields(self) -> dict[str,str]: + val = self.get_custom_field(REDACTOR_FIELD_NAME) + if val: + # assume is json str + fields = json.loads(val) + return fields + + + @property + def is_redacted(self) -> bool: + return self.redacted_fields is not None + + def __str__(self): return f"#{self.id:04d} {self.status.name:<11} {self.priority.name:<6} {self.assigned:<20} {self.subject}" @@ -532,6 +547,7 @@ def get_notes(self, since:dt.datetime|None=None) -> list[TicketNote]: return notes + def get_field(self, fieldname:str): val = getattr(self, fieldname) return val diff --git a/redmine/redmine.py b/redmine/redmine.py index 7c93434..abf0cd3 100644 --- a/redmine/redmine.py +++ b/redmine/redmine.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -"""redmine client""" - import os import re import logging @@ -18,9 +15,8 @@ TIMEOUT = 10 # seconds SYNC_FIELD_NAME = "syncdata" BLOCKED_TEAM_NAME = "blocked" -STATUS_REJECT = 5 # could to status lookup, based on "reject" +STATUS_REJECT = 5 DEFAULT_TRACKER = "External-Comms-Intake" -#TRACKER_REGEX = re.compile(r"tracker=([\w-]+)") TRACKER_REGEX = re.compile(r"\s*\[([\w-]+)\]\s*") @@ -39,9 +35,7 @@ def __init__(self, session:RedmineSession, user_mgr:UserManager, ticket_mgr:Tick self.user_mgr = user_mgr self.ticket_mgr = ticket_mgr - # sanity check - self.validate_sanity() # FATAL if not - + self.validate_sanity() @classmethod def from_session(cls, session:RedmineSession, default_project:int): @@ -67,8 +61,8 @@ def fromenv(cls): def reindex(self): - self.ticket_mgr.reindex() # re-load enumerations (priority, tracker, etc) - self.user_mgr.reindex() # rebuild the user cache + self.ticket_mgr.reindex() + self.user_mgr.reindex() def sanity_check(self) -> dict[str, bool]: @@ -80,14 +74,12 @@ def validate_sanity(self): for subsystem, good in self.sanity_check().items(): log.info(f"- {subsystem}: {good}") if not good: - #log.critical(f"Subsystem {subsystem} not loading correctly.") raise RedmineException(f"Subsystem {subsystem} not loading correctly.") def find_tracker_in_message(self, message:Message) -> NamedId: tracker = self.find_tracker(message.subject) if tracker.name != DEFAULT_TRACKER: - # valid tracker found in subject. strip it. message.subject = TRACKER_REGEX.sub("", message.subject) return tracker @@ -111,8 +103,8 @@ def get_default_tracker(self) -> NamedId: def create_ticket(self, user:User, message:Message) -> Ticket: """ - This is a special case of ticket creation that manages blocked users - and checks for tracker field in message body to set on new ticket. + Create a ticket - NO redaction happens here + Redaction is done in threader before calling this """ project_id = SCN_PROJECT_ID tracker = self.find_tracker_in_message(message) @@ -128,13 +120,24 @@ def create_ticket(self, user:User, message:Message) -> Ticket: return ticket + def unredact_ticket(self, ticket:Ticket) -> Ticket: + """ + Return ticket with original (unredacted) description + The description field already contains original text + """ + # With new architecture: + # - description = original (unredacted) + # - custom field "redacted" = redacted version for Discord + # So unredact just returns the ticket as-is + return ticket + + def find_ticket_from_str(self, string:str) -> Ticket: """parse a ticket number from a string and get the associated ticket""" - # for now, this is a trivial REGEX to match '#nnn' in a string, and return ticket #nnn match = re.search(r'#(\d+)', string) if match: ticket_num = int(match.group(1)) return self.ticket_mgr.get(ticket_num) else: log.debug(f"Unable to match ticket number in: {string}") - return [] + return [] \ No newline at end of file diff --git a/redmine/users.py b/redmine/users.py index 0a05d39..1b7a81a 100644 --- a/redmine/users.py +++ b/redmine/users.py @@ -50,6 +50,7 @@ def cache_user(self, user: User) -> None: def cache_team(self, team: Team) -> None: """add the team to the cache""" self.teams[team.name] = team + self.user_ids[team.id] = team # hack to make teams visible by ID def get(self, user_id:int): @@ -73,6 +74,8 @@ def find(self, name): return self.get(self.discord_ids[name]) if name in self.teams: return self.teams[name] #ugly. put groups in user collection? + if name in self.user_ids: + return self.user_ids[name] # hack to support user-id in find() return None diff --git a/threader-daemon.service b/threader-daemon.service new file mode 100644 index 0000000..b27d6c7 --- /dev/null +++ b/threader-daemon.service @@ -0,0 +1,21 @@ +[Unit] +Description=Threader Daemon - IMAP Email Processor with PII Redaction +After=network.target + +[Service] +Type=simple +User=scn +WorkingDirectory=/home/scn/netbot-redacted +ExecStart=/home/scn/.local/bin/uv run python threader_daemon.py +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal + +# Graceful shutdown +TimeoutStopSec=300 +KillMode=mixed +KillSignal=SIGTERM + +[Install] +WantedBy=multi-user.target diff --git a/threader/imap.py b/threader/imap.py index 33111fd..48cf3b9 100755 --- a/threader/imap.py +++ b/threader/imap.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""IMAP module""" +"""IMAP module - uses remote LLM API for redaction""" import os import logging @@ -18,15 +18,12 @@ from redmine.model import Attachment, Message from redmine import redmine - -# imapclient docs: https://imapclient.readthedocs.io/en/3.0.0/index.html -# source code: https://github.com/mjs/imapclient - +# Import HTTP client instead of local redactor +from redactor.redactor_client import RedactorClient log = logging.getLogger(__name__) -# from https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python class MLStripper(HTMLParser): """strip HTML from a string""" def __init__(self): @@ -50,10 +47,23 @@ def __init__(self): self.passwd = os.getenv('IMAP_PASSWORD') self.redmine:redmine.Client = redmine.Client.fromenv() - # note: not happy with this method of dealing with complex email address - # but I don't see a better way. open to suggestions + redactor_url = os.getenv('REDACTOR_URL') + if redactor_url: + # Initialize HTTP client for remote redaction + try: + self.redactor = RedactorClient(redactor_url) + log.info("Connected to remote LLM API for redaction") + except Exception as e: + log.error(f"Failed to connect to LLM API: {e}") + log.error("Emails will NOT be redacted!") + self.redactor = None + else: + log.warning("Redactor is not configured. If redaction is needed, please configure REDACTOR_URL env var.") + log.warning("Emails will NOT be redacted!") + self.redactor = None + + def parse_email_address(self, email_addr): - #regex_str = r"(.*)<(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)>" regex_str = r"(.*)<(.*)>" m = re.match(regex_str, email_addr) first = last = addr = "" @@ -63,35 +73,25 @@ def parse_email_address(self, email_addr): first, last = name.rsplit(None, 1) else: first = name - last = "-" # empty string breaks redmine + last = "-" addr = m.group(2) - return first, last, addr else: - # assume it's just email - #log.error(f"Unable to parse email str: {email_addr}") return "", "", email_addr - - def is_html_doc(self, payload: str) -> bool: - # check the first few chars to see if they contain any HTML tags tags = [ "", "" ] head = payload[:20].strip().lower() for tag in tags: if head.startswith(tag): return True - # no html tag found return False - def parse_message(self, data): - # NOTE this policy setting is important, default is "compat-mode" amd we need "default" root = email.message_from_bytes(data, policy=email.policy.default) from_address = root.get("From") subject = root.get("Subject") - # ticket-485: capture to and cc headers to_header = root.get("To") cc_header = root.get("Cc") message = Message(from_address, subject, to_header, cc_header) @@ -105,15 +105,12 @@ def parse_message(self, data): content_type=content_type, payload=part.get_payload(decode=True))) log.debug(f"Added attachment: {part.get_filename()} {content_type}") - elif content_type == 'text/plain': # FIXME std const? + elif content_type == 'text/plain': payload = part.get_payload(decode=True).decode('UTF-8') - # http://10.10.0.218/issues/208 if payload == "": payload = root.get_body().get_content() - # search for HTML if self.is_html_doc(payload): - # strip HTML payload = self.strip_html_tags(payload) log.debug(f"HTML payload after: {payload}") @@ -141,27 +138,21 @@ def strip_html_tags(self, text:str) -> str: "1600 Amphitheatre Pkwy", "Mountain View CA 94043 USA", ] + def strip_forwards(self, text:str) -> str: - # strip any forwarded messages - # from ^> forward_tag = "------ Forwarded message ---------" idx = text.find(forward_tag) if idx > -1: text = text[0:idx] - # search for "On ... wrote:" p = re.compile(r"^On .* <.*>\s+wrote:", flags=re.MULTILINE|re.DOTALL|re.IGNORECASE) match = p.search(text) if match: text = text[0:match.start()] - # TODO search for -- - - # look for google content, as in http://10.10.0.218/issues/323 buffer = "" for line in text.splitlines(): skip = False - # search for skip_strs for skip_str in self.skip_strs: if line.startswith(skip_str): skip = True @@ -178,94 +169,239 @@ def handle_message(self, msg_id:str, message:Message): subject = message.subject_cleaned() log.debug(f'uid:{msg_id} - from:{last}, {first}, email:{addr}, subject:{subject}') - ticket = None - # first, search for a matching subject - tickets = self.redmine.ticket_mgr.match_subject(subject) - if len(tickets) == 1: - # as expected - ticket = tickets[0] - log.debug(f"found ticket id={ticket.id} for subject: {subject}") - elif len(tickets) >= 2: - # more than expected - log.warning(f"subject query returned {len(tickets)} results, using first: {subject}") - ticket = tickets[0] - - # next, find ticket using the subject, if possible - if ticket is None: - # this uses a simple REGEX '#\d+' to match ticket numbers - ticket = self.redmine.find_ticket_from_str(subject) + # Redact message using remote API + original_note = message.note + + if self.redactor: + try: + log.info(f"Redacting PII from message {msg_id} via LLM API...") + redacted = self.redactor.redact_text(original_note) + log.info(f"Redaction complete for message {msg_id}") + except Exception as e: + log.error(f"Redaction failed for message {msg_id}: {e}") + log.warning("Creating ticket WITHOUT redaction") + redacted = None + else: + log.warning("No redactor available, creating ticket WITHOUT redaction") + redacted = None - # get user id from from_address + # Get user user = self.redmine.user_mgr.get_by_name(addr) if user is None: log.debug(f"Unknown email address, no user found: {addr}, {message.from_address}") - # create new user user = self.redmine.user_mgr.create(addr, first, last, user_login=None) - log.info(f"Unknow user: {addr}, created new account.") - # make sure user is in users group + log.info(f"Unknown user: {addr}, created new account.") + self.redmine.user_mgr.join_team(user, "users") - # upload any attachments + # Upload attachments self.redmine.ticket_mgr.upload_attachments(user, message.attachments) + # Check for existing ticket + tickets = self.redmine.ticket_mgr.match_subject(subject) + + if len(tickets) == 1: + ticket = tickets[0] + log.debug(f"found ticket id={ticket.id} for subject: {subject}") + elif len(tickets) >= 2: + log.warning(f"subject query returned {len(tickets)} results, using first: {subject}") + ticket = tickets[0] + else: + ticket = self.redmine.find_ticket_from_str(subject) + if ticket: - # found a ticket, append the message - self.redmine.ticket_mgr.append_message(ticket.id, user.login, message.note, message.attachments) + # Update existing ticket + # self.redmine.ticket_mgr.append_message(ticket.id, user.login, message.note, message.attachments) TODO: CHANGE LATER 2/9 + # Use API key account (admin) instead of impersonating sender + # This avoids 403 permission errors for external users + attributed_note = f"**From:** {user.name} ({user.mail})\n\n{message.note}" + self.redmine.ticket_mgr.append_message(ticket.id, None, attributed_note, message.attachments) log.info(f"Updated ticket #{ticket.id} with message from {user.login} and {len(message.attachments)} attachments") else: - # no open tickets, create new ticket for the email message - ticket = self.redmine.create_ticket(user, message) + if redacted: + # Store REDACTED in description (public facing) + message.note = redacted.text + ticket = self.redmine.create_ticket(user, message) + + # Store ORIGINAL in unredacted custom field (PII admin only) + unredacted_cf = self.redmine.ticket_mgr.get_custom_field("unredacted") + if unredacted_cf: + fields = { + "custom_fields": [ + {"id": unredacted_cf.id, "value": original_note} + ] + } + self.redmine.ticket_mgr.update(ticket.id, fields) + log.info(f"Stored original in unredacted CF for ticket #{ticket.id}") + else: + log.error("Custom field 'unredacted' not found!") + else: + # No redaction available, store original in description only + message.note = original_note + ticket = self.redmine.create_ticket(user, message) log.info(f"Created new ticket for: {ticket}, with {len(message.attachments)} attachments") - def synchronize(self): + """Process ONE email, then return. Returns number of emails processed (0 or 1).""" + processed_count = 0 try: with IMAPClient(host=self.host, ssl=True) as server: - # https://imapclient.readthedocs.io/en/3.0.1/api.html#imapclient.IMAPClient.oauthbearer_login - # NOTE: self.user -> IMAP_USER -> identity, self.user -> IMAP_PASSWD -> token server.login(self.user, self.passwd) - #server.oauthbearer_login(self.user, self.passwd) - #server.oauth2_login(self.user, self.passwd) - server.select_folder("INBOX", readonly=False) log.info(f'logged into imap {self.host}') messages = server.search("UNSEEN") log.info(f"processing {len(messages)} new messages from {self.host}") - for uid, message_data in server.fetch(messages, "RFC822").items(): - # process each message returned by the query - try: - # decode the message - data = message_data[b"RFC822"] - message = self.parse_message(data) - - # handle the message - self.handle_message(uid, message) + if not messages: + # No emails to process + log.info("done. processed 0 messages") + return 0 + + # Process ONLY the first email + uid = messages[0] # Get first unread email + message_data = server.fetch([uid], "RFC822") + + try: + data = message_data[uid][b"RFC822"] + message = self.parse_message(data) + self.handle_message(uid, message) + server.add_flags(uid, [SEEN, DELETED]) + processed_count = 1 + log.info("done. processed 1 message") + except Exception as e: + log.error(f"Message {uid} can not be processed: {e}") + traceback.print_exc() + with open(f"message-err-{uid}.eml", "wb") as file: + file.write(data) + server.add_flags(uid, [SEEN]) + processed_count = 0 - # mark msg uid seen and deleted, as per redmine imap.rb - server.add_flags(uid, [SEEN, DELETED]) - - except Exception as e: - log.error(f"Message {uid} can not be processed: {e}") - traceback.print_exc() - # save the message data in a file - with open(f"message-err-{uid}.eml", "wb") as file: - file.write(data) - server.add_flags(uid, [SEEN]) - - log.info(f"done. processed {len(messages)} messages") except Exception as ex: log.error(f"caught exception syncing IMAP: {ex}") traceback.print_exc() + return processed_count + + # def process_edit_job(self, job: dict): + # #Process a ticket edit job from the queue + # from redaction_queue import RedactionQueue + + # ticket_id = job["ticket_id"] + # new_description = job["description"] + # user_info = job["user"] + + # log.info(f"Processing edit for ticket #{ticket_id}") + + # queue = RedactionQueue() + + # try: + # # Lock the ticket + # queue.lock_ticket(ticket_id, "edit") + + # # Get ticket + # ticket = self.redmine.ticket_mgr.get(ticket_id) + # if not ticket: + # raise Exception(f"Ticket #{ticket_id} not found") + + # # Redact the new description + # if self.redactor: + # log.info(f"Redacting new description for ticket #{ticket_id}") + # redacted = self.redactor.redact_text(new_description) + # log.info(f"Redaction complete for ticket #{ticket_id}") + # else: + # log.warning("No redactor available") + # redacted = None + + # # Update ticket in Redmine + # if redacted: + # # Store ORIGINAL in description (admin-only) + # fields = {"description": new_description} + # self.redmine.ticket_mgr.update(ticket_id, fields) + + # # Store REDACTED in custom field (public) + # redacted_cf = self.redmine.ticket_mgr.get_custom_field("redacted") + # if redacted_cf: + # fields = { + # "custom_fields": [ + # {"id": redacted_cf.id, "value": redacted.text} + # ] + # } + # self.redmine.ticket_mgr.update(ticket_id, fields) + # log.info(f"Updated redacted field for ticket #{ticket_id}") + # else: + # # No redaction, just update description + # fields = {"description": new_description} + # self.redmine.ticket_mgr.update(ticket_id, fields) + + # log.info(f"Successfully updated ticket #{ticket_id}") + + # finally: + # # Always unlock ticket + # queue.unlock_ticket(ticket_id) + + + def process_edit_job(self, job: dict): + #process a ticket edit job from the queue + from redaction_queue import RedactionQueue + + ticket_id = job["ticket_id"] + new_description = job["description"] + user_info = job.get("user", {}) + + queue = RedactionQueue() + + try: + queue.lock_ticket(ticket_id, "edit") + + log.info(f"Processing edit for ticket #{ticket_id}") + + # Get ticket + # ticket = self.redmine.ticket_mgr.get(ticket_id) + # Only ticket_id is needed. + + # Redact description + log.info(f"Redacting new description for ticket #{ticket_id}") + if self.redactor: + redacted = self.redactor.redact_text(new_description) + else: + redacted = None + + # Update Redmine + if redacted: + # Redacted in description (public facing) + # Original in unredacted CF (PII admin only) + unredacted_cf = self.redmine.ticket_mgr.get_custom_field("unredacted") + if unredacted_cf: + fields = { + "description": redacted.text, + "custom_fields": [ + {"id": unredacted_cf.id, "value": new_description} + ], + "notes": f"Ticket description updated and redacted by {user_info.get('name', 'user')}" + } + self.redmine.ticket_mgr.update(ticket_id, fields) + log.info(f"Updated description (redacted) and unredacted CF for ticket #{ticket_id}") + else: + log.error("Custom field 'unredacted' not found!") + self.redmine.ticket_mgr.update(ticket_id, { + "description": redacted.text, + "notes": f"Ticket description updated and redacted by {user_info.get('name', 'user')}" + }) + else: + # No redaction available - store original in description + params = { + "description": new_description, + "notes": f"Ticket description updated (no redaction) by {user_info.get('name', 'user')}" + } + self.redmine.ticket_mgr.update(ticket_id, params) + log.info(f"Successfully updated ticket #{ticket_id}") + + finally: + queue.unlock_ticket(ticket_id) + -# Run the IMAP sync process if __name__ == '__main__': log.info('initializing IMAP threader') - - # load credentials load_dotenv() - - # construct the client and run the email check - Client().synchronize() + Client().synchronize() \ No newline at end of file diff --git a/threader_daemon.py b/threader_daemon.py new file mode 100644 index 0000000..a798fec --- /dev/null +++ b/threader_daemon.py @@ -0,0 +1,162 @@ + +# import asyncio +# import logging +# import signal +# import sys +# from pathlib import Path +# from dotenv import load_dotenv + +# # Add netbot-redacted to path +# sys.path.insert(0, str(Path(__file__).parent)) + +# from threader.imap import Client # Changed from IMAPClient to Client + +# logging.basicConfig( +# level=logging.INFO, +# format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +# ) +# log = logging.getLogger(__name__) + +# shutdown_requested = False + +# def signal_handler(sig, frame): +# global shutdown_requested +# log.info(f"Received signal {sig}, initiating graceful shutdown...") +# shutdown_requested = True + +# async def main(): +# global shutdown_requested + +# # Register signal handlers +# signal.signal(signal.SIGTERM, signal_handler) +# signal.signal(signal.SIGINT, signal_handler) + +# log.info("Starting Threader Daemon") +# log.info("Processes one email completely, then checks for next") + +# # Load environment variables +# load_dotenv() + +# # Create client once +# client = Client() # Changed from IMAPClient() + +# while not shutdown_requested: +# try: +# log.info("Checking IMAP for new messages...") + +# # Process emails - this handles ONE email at a time +# # Returns number of emails processed +# processed_count = client.synchronize() + +# if processed_count > 0: +# log.info(f"Processed {processed_count} email(s)") +# # Immediately check for next email (no delay after processing) +# continue +# else: +# # No emails found - wait 60 seconds before checking again +# log.info("No new emails. Waiting 60 seconds before next check...") +# await asyncio.sleep(60) + +# except KeyboardInterrupt: +# log.info("Keyboard interrupt received") +# break +# except Exception as e: +# log.error(f"Error in main loop: {e}", exc_info=True) +# # Wait before retrying on error +# await asyncio.sleep(60) + +# log.info("Threader Daemon stopped") + +# if __name__ == "__main__": +# asyncio.run(main()) + + +#!/usr/bin/env python3 +import asyncio +import logging +import signal +import sys +from pathlib import Path +from dotenv import load_dotenv + +# Add netbot-redacted to path +sys.path.insert(0, str(Path(__file__).parent)) + +from threader.imap import Client +from redaction_queue import RedactionQueue + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +log = logging.getLogger(__name__) + +shutdown_requested = False + +def signal_handler(sig, frame): + global shutdown_requested + log.info(f"Received signal {sig}, initiating graceful shutdown...") + shutdown_requested = True + +async def main(): + global shutdown_requested + + # Register signal handlers + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + log.info("Starting Threader Daemon") + log.info("Processes emails and edit requests sequentially") + + # Load environment variables + load_dotenv() + + # Create client and queue manager + client = Client() + queue = RedactionQueue() + + while not shutdown_requested: + try: + # Check for edit jobs in queue FIRST (priority) + edit_job = queue.get_next_job() + + if edit_job: + log.info(f"Processing edit job: {edit_job['id']}") + queue.mark_processing(edit_job['id']) + + try: + # Process the edit job + client.process_edit_job(edit_job) + queue.mark_complete(edit_job['id']) + log.info(f"Completed edit job: {edit_job['id']}") + except Exception as e: + log.error(f"Edit job failed: {e}", exc_info=True) + queue.mark_failed(edit_job['id'], str(e)) + + # Immediately check for next job + continue + + # No edit jobs, check IMAP + log.info("Checking IMAP for new messages...") + processed_count = client.synchronize() + + if processed_count > 0: + log.info(f"Processed {processed_count} email(s)") + # Immediately check for next job + continue + else: + # No work to do - wait 60 seconds + log.info("No pending jobs. Waiting 60 seconds before next check...") + await asyncio.sleep(60) + + except KeyboardInterrupt: + log.info("Keyboard interrupt received") + break + except Exception as e: + log.error(f"Error in main loop: {e}", exc_info=True) + await asyncio.sleep(60) + + log.info("Threader Daemon stopped") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file