Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
750 commits
Select commit Hold shift + click to select a range
5abb2f4
update
jaredmauch May 15, 2025
9a09459
update
jaredmauch May 15, 2025
2d1d30d
update
jaredmauch May 15, 2025
b653968
update
jaredmauch May 15, 2025
a95ea17
update
jaredmauch May 15, 2025
30cad35
update
jaredmauch May 15, 2025
1e7d327
update
jaredmauch May 15, 2025
7bf6ae4
update
jaredmauch May 15, 2025
21df7d7
update
jaredmauch May 15, 2025
44ff079
update
jaredmauch May 15, 2025
ec6c62d
update
jaredmauch May 15, 2025
8d2a1ae
update
jaredmauch May 15, 2025
1eb6409
update
jaredmauch May 15, 2025
eb53983
update
jaredmauch May 15, 2025
50f4159
update
jaredmauch May 15, 2025
abd905e
update
jaredmauch May 15, 2025
3c68164
update
jaredmauch May 15, 2025
6c8df41
update
jaredmauch May 15, 2025
8f8b0db
update
jaredmauch May 15, 2025
0dcefa6
update
jaredmauch May 15, 2025
62cb018
update
jaredmauch May 15, 2025
48032eb
update
jaredmauch May 15, 2025
21dc531
update
jaredmauch May 15, 2025
730d1d5
update
jaredmauch May 15, 2025
68bae7c
update
jaredmauch May 15, 2025
6a7b388
update
jaredmauch May 15, 2025
39a0d7d
update
jaredmauch May 15, 2025
1dab55e
update
jaredmauch May 15, 2025
94311fb
update
jaredmauch May 15, 2025
68a8a07
update
jaredmauch May 15, 2025
d43f079
update
jaredmauch May 15, 2025
322a8fb
update
jaredmauch May 15, 2025
794dc64
update
jaredmauch May 15, 2025
ab11f98
update
jaredmauch May 15, 2025
88576fe
update
jaredmauch May 15, 2025
7ff2c5a
update
jaredmauch May 15, 2025
8c3bc9f
update
jaredmauch May 15, 2025
327b51a
update
jaredmauch May 15, 2025
fc93537
update
jaredmauch May 15, 2025
3aeeabb
update
jaredmauch May 15, 2025
d3f6b08
update
jaredmauch May 15, 2025
ba3133d
update
jaredmauch May 15, 2025
fdfe4a7
update
jaredmauch May 15, 2025
0df3716
update
jaredmauch May 15, 2025
707a191
update
jaredmauch May 15, 2025
9319721
update
jaredmauch May 15, 2025
eae7dc6
update
jaredmauch May 15, 2025
bfd5c76
update
jaredmauch May 15, 2025
2948dcc
update
jaredmauch May 15, 2025
aa2a96f
update
jaredmauch May 15, 2025
b6bb880
update
jaredmauch May 15, 2025
d944958
update
jaredmauch May 15, 2025
70277f2
update
jaredmauch May 15, 2025
bfc10e3
update
jaredmauch May 15, 2025
5b688fc
update
jaredmauch May 15, 2025
dc04743
update
jaredmauch May 15, 2025
3e8ccf5
Fix recipient handling in queue submissions to prevent 'No recipients…
jaredmauch May 15, 2025
5e67c01
update
jaredmauch May 15, 2025
7f9ba76
Ensure all scripts and inject logic enqueue Mailman.Message.Message o…
jaredmauch May 15, 2025
11febe8
Ensure all scripts enqueue messages in Mailman.Message.Message format
jaredmauch May 15, 2025
b7f32d0
Fix list locking in CommandRunner to prevent NotLockedError
jaredmauch May 15, 2025
078f470
Fix missing recipient in command response messages
jaredmauch May 15, 2025
b841d65
update
jaredmauch May 15, 2025
8343c81
Fix recipient handling in UserNotification._enqueue to ensure recipie…
jaredmauch May 15, 2025
15adb5c
Fix duplicate msgdata in UserNotification._enqueue
jaredmauch May 15, 2025
2cc0010
Fix bounce processing next action time to properly update after proce…
jaredmauch May 15, 2025
2c65098
Fix BounceMixin initialization to set next_action time to 1 hour in t…
jaredmauch May 15, 2025
86369fa
update
jaredmauch May 15, 2025
151689d
update
jaredmauch May 15, 2025
8865275
update
jaredmauch May 15, 2025
160d375
update
jaredmauch May 15, 2025
28164d3
update
jaredmauch May 15, 2025
2bed767
update
jaredmauch May 15, 2025
b6b280d
update
jaredmauch May 15, 2025
aa25401
update
jaredmauch May 15, 2025
5ef61b7
update
jaredmauch May 15, 2025
0b2b156
update
jaredmauch May 15, 2025
7a2f05a
update
jaredmauch May 15, 2025
8ff0e05
update
jaredmauch May 15, 2025
b2480fa
update
jaredmauch May 15, 2025
0ecbce5
update
jaredmauch May 15, 2025
fcab222
update
jaredmauch May 15, 2025
56178ea
update
jaredmauch May 15, 2025
86b53cc
update
jaredmauch May 15, 2025
d556f3e
update
jaredmauch May 15, 2025
69dc206
update
jaredmauch May 15, 2025
dbb545d
update
jaredmauch May 15, 2025
c5846bb
update
jaredmauch May 15, 2025
a4a6ad6
update
jaredmauch May 15, 2025
04f59ca
update
jaredmauch May 15, 2025
646fe3a
update
jaredmauch May 15, 2025
8a1c5f5
update
jaredmauch May 15, 2025
f9ca094
update
jaredmauch May 15, 2025
7456a2b
update
jaredmauch May 15, 2025
339519a
MemberAdaptor
jaredmauch May 15, 2025
bd545fa
MemberAdaptor
jaredmauch May 15, 2025
689daa8
update
jaredmauch May 15, 2025
94fc278
update
jaredmauch May 15, 2025
114bc80
Improve file handling and cleanup in Runner and Switchboard classes
jaredmauch May 15, 2025
e940e5e
Reduce logging verbosity in Runner class
jaredmauch May 15, 2025
5c5ab91
Fix file counter in Runner logging
jaredmauch May 15, 2025
dc7de8c
Fix lock transfer race condition in mailmanctl start by using atomic …
jaredmauch May 15, 2025
1007575
Fix header handling in CookHeaders.py to ensure proper bounce processing
jaredmauch May 15, 2025
7b2ed9a
update
jaredmauch May 15, 2025
e92da08
Fix character encoding issues in UserNotification class - Prevent ASC…
jaredmauch May 15, 2025
a88cc4c
remove check
jaredmauch May 19, 2025
4c22d97
attempted admindb fix
jaredmauch May 19, 2025
78ddd51
update
jaredmauch May 19, 2025
c7b9300
update
jaredmauch May 19, 2025
5419686
update
jaredmauch May 19, 2025
7f77552
update
jaredmauch May 19, 2025
2ba65c1
update
jaredmauch May 19, 2025
2ba2f93
update
jaredmauch May 19, 2025
4763877
update
jaredmauch May 19, 2025
42a714a
update
jaredmauch May 19, 2025
ccbc954
update
jaredmauch May 19, 2025
e0f8821
update
jaredmauch May 19, 2025
0d32a9a
update
jaredmauch May 19, 2025
fc7c09c
update
jaredmauch May 19, 2025
f626d97
update
jaredmauch May 19, 2025
d874c20
update
jaredmauch May 19, 2025
6807a49
update
jaredmauch May 19, 2025
9216755
update
jaredmauch May 19, 2025
f8fe880
update
jaredmauch May 19, 2025
8889668
update
jaredmauch May 19, 2025
f24fe88
update
jaredmauch May 19, 2025
42eb97d
update
jaredmauch May 19, 2025
454b027
update
jaredmauch May 19, 2025
3e14923
update
jaredmauch May 19, 2025
52c8980
update
jaredmauch May 20, 2025
486c51c
update
jaredmauch May 20, 2025
0c139ba
update
jaredmauch May 20, 2025
8bec7ac
update
jaredmauch May 20, 2025
e9293cf
update
jaredmauch May 20, 2025
c4c03dc
update
jaredmauch May 20, 2025
a8023b2
update
jaredmauch May 20, 2025
b6b934e
update
jaredmauch May 20, 2025
cbb7692
update
jaredmauch May 20, 2025
225e997
update
jaredmauch May 20, 2025
56b31aa
update
jaredmauch May 20, 2025
695017f
update
jaredmauch May 20, 2025
6857bd2
update
jaredmauch May 20, 2025
db63723
update
jaredmauch May 20, 2025
175f9e3
update for python3
jaredmauch May 22, 2025
864aec7
Modernize HTML generation in Python codebase and templates to use HTM…
jaredmauch May 22, 2025
80c3e2c
Fix TypeError in admin.py: convert breadcrumb items to strings before…
jaredmauch May 22, 2025
f91e134
update
jaredmauch May 22, 2025
e3fc5ce
update
jaredmauch May 22, 2025
6a0b6d5
update
jaredmauch May 22, 2025
cd7f677
update
jaredmauch May 22, 2025
a4c4f5b
update
jaredmauch May 22, 2025
7bc2677
update
jaredmauch May 22, 2025
34badea
update
jaredmauch May 22, 2025
dfe9660
Fix HTML entity handling in Utils.py and related files to prevent str…
jaredmauch May 22, 2025
44971dd
Fix Unicode encoding issues in UserNotification class to properly han…
jaredmauch May 22, 2025
1480bca
Fix UnboundLocalError in private.py by using list's preferred languag…
jaredmauch May 22, 2025
23508d6
Improve global_options function in options.py with better error handl…
jaredmauch May 22, 2025
ba6e124
Fix reCAPTCHA error message formatting in subscribe.py
jaredmauch May 22, 2025
ccf82f7
Fix subscription data handling to use tuple format consistently
jaredmauch May 22, 2025
48441e3
Enhance Exchange bounce detector to better handle Office 365 format
jaredmauch May 22, 2025
49ae0eb
Fix template interpolation in admlogin.html to use % style formatting
jaredmauch May 22, 2025
9fccab3
Fix template interpolation in admlogin.html to use % style formatting
jaredmauch May 22, 2025
d42e0c1
Fix queue file handling to prevent truncation and improve reliability
jaredmauch May 22, 2025
ad47d13
Enhance OutgoingRunner error handling and logging to better diagnose …
jaredmauch May 22, 2025
7d29d5a
Reduce log noise by changing delivery module import messages to trace…
jaredmauch May 22, 2025
d0df5aa
Reduce log noise by changing standard replacements message to trace l…
jaredmauch May 22, 2025
6bb4fee
Add message counting and enhanced error tracking to OutgoingRunner
jaredmauch May 22, 2025
dce6217
Add process coordination to prevent multiple OutgoingRunner instances
jaredmauch May 22, 2025
beeab0f
Fix OutgoingRunner main loop and process coordination
jaredmauch May 23, 2025
a10ebd3
Reduce OutgoingRunner instances to 1 since we now have process coordi…
jaredmauch May 23, 2025
7b3926a
Enhanced OutgoingRunner with improved error handling, logging, and me…
jaredmauch May 23, 2025
b26d765
Enhanced CommandRunner and VirginRunner with improved no-files handli…
jaredmauch May 23, 2025
f5111b8
Enhanced admin page to properly display categories and subcategories
jaredmauch May 23, 2025
562d0c8
Enhanced Runner logging to include specific runner class names in log…
jaredmauch May 23, 2025
6389f02
Implemented exponential backoff for runners when no files to process
jaredmauch May 23, 2025
203e582
Removed unnecessary debug logging from OutgoingRunner
jaredmauch May 23, 2025
f635e9e
Modified mailmanctl to respect runner backoff behavior
jaredmauch May 23, 2025
8093aa2
Optimized Runner to use os.stat() for directory change detection
jaredmauch May 23, 2025
edb034b
Fixed _snooze method to properly handle filecnt and maintain original…
jaredmauch May 23, 2025
8492d9b
Modified mailmanctl to properly daemonize processes
jaredmauch May 23, 2025
00916d4
Restored original mailmanctl from mailman-2.1.39
jaredmauch May 23, 2025
f927dbc
update
jaredmauch May 23, 2025
c8e9970
Remove max-width: 800px; from all template files to allow full page w…
jaredmauch May 24, 2025
188ecbf
Update string handling for Python 3 compatibility in logging and path…
jaredmauch May 24, 2025
fda3d55
Reduce debug logging in admin.py to make logs cleaner and more focused
jaredmauch May 25, 2025
323159d
Fix: Add back GetConfigInfo call in show_variables function
jaredmauch May 25, 2025
2914bee
Fix: Improve error handling for non-existent lists in listinfo.py
jaredmauch May 25, 2025
f63a5a3
Fix: Restore original HTML formatting in listinfo.py while keeping im…
jaredmauch May 25, 2025
ebcc9ad
Fix: Properly handle binary strings in membership adapter for Python …
jaredmauch May 25, 2025
a207dbb
Fix: Remove b'...' prefix from member names in admin interface
jaredmauch May 25, 2025
e74f54e
Fix: Properly handle binary strings in membership adapter for Python …
jaredmauch May 25, 2025
5978168
Fix: Ensure proper string handling when loading pickle files from Pyt…
jaredmauch May 25, 2025
2beb354
Fix: Add digestable property to OldStyleMemberships class
jaredmauch May 25, 2025
8fb7a3c
Add missing digest-related properties to OldStyleMemberships class
jaredmauch May 25, 2025
7bc6631
Add setters for digest-related properties in OldStyleMemberships class
jaredmauch May 25, 2025
5455bbf
Fix argument parsing and error handling in bin/update script
jaredmauch May 25, 2025
b1b987d
Fix lock handling in bin/update script to properly acquire and releas…
jaredmauch May 25, 2025
359f858
Fix: Improve lock handling safety in bin/update to prevent data loss
jaredmauch May 25, 2025
3a56794
Fix: Handle stale locks in bin/update by adding force unlock capability
jaredmauch May 25, 2025
5d26c99
Add pickle protocol version reporting to bin/update
jaredmauch May 25, 2025
0e04fc4
Add pickle protocol version reporting for both loading and saving files
jaredmauch May 25, 2025
e8df15c
Add pickle protocol version reporting when loading files
jaredmauch May 25, 2025
ffc2e74
Improve pickle loading robustness and error handling in MailList.py
jaredmauch May 25, 2025
649e7cd
Improve pickle loading for protocol 4 files and add detailed error lo…
jaredmauch May 25, 2025
6294a39
Improve pickle protocol 2 file handling and add detailed error reporting
jaredmauch May 25, 2025
c47d317
Update MailList.py to use new pickle utility functions
jaredmauch May 25, 2025
8b6bd74
Fix: Add C_ function definition to resolve pickle loading errors
jaredmauch May 25, 2025
6f70c3c
Fix: Properly import all utility functions from Utils.py
jaredmauch May 25, 2025
a7bf2a5
Fix: Add pickle utility functions to Utils.py
jaredmauch May 25, 2025
646a60b
Fix: Add missing pickle import to Utils.py
jaredmauch May 25, 2025
c726d97
Improve pickle file handling in bin/update to match Mailman 2.1.39 be…
jaredmauch May 25, 2025
a0ff423
Add fix_imports=True and encoding='latin1' to pickle.load in bin/chec…
jaredmauch May 25, 2025
642baf8
Update pickle loading in bin/check_db to use fix_imports=True and enc…
jaredmauch May 25, 2025
476e92d
Improve Python 2 compatibility in bin/check_db by trying UTF-8 then l…
jaredmauch May 25, 2025
9f67bda
Remove unused load_pickle function from bin/check_db
jaredmauch May 25, 2025
a34f1d4
Update pickle protocol detection to use loaded data instead of file h…
jaredmauch May 25, 2025
5f740a4
Update pickle protocol detection in bin/update to match bin/check_db
jaredmauch May 25, 2025
3095bc3
Fix get_pickle_protocol to properly detect protocol version by loadin…
jaredmauch May 25, 2025
f1158d8
Fix error message formatting in MailList.py to handle error messages …
jaredmauch May 25, 2025
5f2368e
Fix get_pickle_protocol to properly detect protocol version from file…
jaredmauch May 25, 2025
c4070cc
Major Python 3 migration and bug fixes
Nov 12, 2025
a5eb9dd
Fix KeyError in HyperDatabase when checking parentID in articleIndex
Nov 12, 2025
a56df58
Fix TypeError: Strings must be encoded before hashing
Nov 12, 2025
e092585
Fix TypeError: find() argument 1 must be str, not bytes
Nov 12, 2025
1c69b6b
Fix pipeline attribute handling in IncomingRunner._get_pipeline
Nov 12, 2025
8e27a47
Add release note for pipeline attribute fix
Nov 12, 2025
f0dbedd
Squashed divergence period: Python 3 migration and fixes
Nov 12, 2025
d5e801c
Remove temporary script
Nov 12, 2025
f1e7ec6
Major Python 3 migration and bug fixes
Nov 12, 2025
9cd7614
Fix KeyError in HyperDatabase when checking parentID in articleIndex
Nov 12, 2025
a13a35d
Fix TypeError: Strings must be encoded before hashing
Nov 12, 2025
51dbefb
Fix TypeError: find() argument 1 must be str, not bytes
Nov 12, 2025
b14ae24
Fix pipeline attribute handling in IncomingRunner._get_pipeline
Nov 12, 2025
58a2332
Add release note for pipeline attribute fix
Nov 12, 2025
c82bcbf
Merge pull request #19 from jaredmauch/collapse-divergence
jaredmauch Nov 12, 2025
8b0024f
Upgrade password hashing from SHA1 to PBKDF2-SHA256
Nov 12, 2025
26ba1dc
Add password upgrade notifications and dry-run option to bin/update
Nov 12, 2025
34c2145
Fix duplicate list processing in password upgrade check
Nov 12, 2025
66840a7
Handle permission errors gracefully during password auto-upgrade
Nov 12, 2025
c79388b
Improve duplicate list detection in password upgrade check
Nov 12, 2025
3ec88f8
Fix list name display in password upgrade notifications
Nov 12, 2025
d4787c2
update
Nov 13, 2025
d91509c
Remove MD5 and crypt() password downgrade support, only support SHA1 …
Nov 13, 2025
4c4925b
Fix build dependencies for scripts copied to build directory
Nov 13, 2025
9b9699b
Fix .po file dependencies to prevent unnecessary rebuilds
Nov 13, 2025
e777629
Merge pull request #20 from jaredmauch/hashdigest_fixes
jaredmauch Nov 13, 2025
7369fda
- use @PYTHON@
zoulasc Nov 23, 2025
664ffc4
use @PYTHON@
zoulasc Nov 23, 2025
ce0cffc
- Use @PYTHON@ and generate the script
zoulasc Nov 23, 2025
963778e
make the cgi stuff work by:
zoulasc Nov 23, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
mailman.po~
Mailman/Archiver/Makefile
Mailman/Bouncers/Makefile
Mailman/Cgi/Makefile
Mailman/Commands/Makefile
Mailman/Defaults.py
Mailman/Gui/Makefile
Mailman/Handlers/Makefile
Mailman/Logging/Makefile
Mailman/MTA/Makefile
Mailman/Makefile
Mailman/Queue/Makefile
Mailman/__pycache__/
Mailman/mm_cfg.py.dist
Makefile
bin/Makefile
build/
config.log
config.status
cron/Makefile
cron/crontab.in
messages/Makefile
messages/ar/LC_MESSAGES/mailman.mo
messages/ast/LC_MESSAGES/mailman.mo
messages/ca/LC_MESSAGES/mailman.mo
messages/cs/LC_MESSAGES/mailman.mo
messages/da/LC_MESSAGES/mailman.mo
messages/de/LC_MESSAGES/mailman.mo
messages/el/LC_MESSAGES/mailman.mo
messages/eo/LC_MESSAGES/mailman.mo
messages/es/LC_MESSAGES/mailman.mo
messages/et/LC_MESSAGES/mailman.mo
messages/eu/LC_MESSAGES/mailman.mo
messages/fa/LC_MESSAGES/mailman.mo
messages/fi/LC_MESSAGES/mailman.mo
messages/fr/LC_MESSAGES/mailman.mo
messages/gl/LC_MESSAGES/mailman.mo
messages/he/LC_MESSAGES/mailman.mo
messages/hr/LC_MESSAGES/mailman.mo
messages/hu/LC_MESSAGES/mailman.mo
messages/ia/LC_MESSAGES/mailman.mo
messages/it/LC_MESSAGES/mailman.mo
messages/ja/LC_MESSAGES/mailman.mo
messages/ko/LC_MESSAGES/mailman.mo
messages/lt/LC_MESSAGES/mailman.mo
messages/nl/LC_MESSAGES/mailman.mo
messages/no/LC_MESSAGES/mailman.mo
messages/pl/LC_MESSAGES/mailman.mo
messages/pt/LC_MESSAGES/mailman.mo
messages/pt_BR/LC_MESSAGES/mailman.mo
messages/ro/LC_MESSAGES/mailman.mo
messages/ru/LC_MESSAGES/mailman.mo
messages/sk/LC_MESSAGES/mailman.mo
messages/sl/LC_MESSAGES/mailman.mo
messages/sr/LC_MESSAGES/mailman.mo
messages/sv/LC_MESSAGES/mailman.mo
messages/tr/LC_MESSAGES/mailman.mo
messages/uk/LC_MESSAGES/mailman.mo
messages/vi/LC_MESSAGES/mailman.mo
messages/zh_CN/LC_MESSAGES/mailman.mo
messages/zh_TW/LC_MESSAGES/mailman.mo
misc/Makefile
misc/mailman
misc/paths.py
scripts/Makefile
src/Makefile
src/admin
src/admindb
src/common.o
src/confirm
src/create
src/edithtml
src/listinfo
src/mailman
src/options
src/private
src/rmlist
src/roster
src/subscribe
src/vsnprintf.o
templates/Makefile
templates/.converted.stamp
tests/Makefile
tests/bounces/Makefile
tests/msgs/Makefile
messages/.converted.stamp
36 changes: 31 additions & 5 deletions Mailman/Archiver/Archiver.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import errno
import traceback
import re
from io import StringIO
import tempfile

from Mailman import mm_cfg
from Mailman import Mailbox
Expand Down Expand Up @@ -150,7 +150,7 @@ def __archive_file(self, afn):
"""Open (creating, if necessary) the named archive file."""
omask = os.umask(0o002)
try:
return Mailbox.Mailbox(open(afn, 'a+'))
return Mailbox.Mailbox(open(afn, 'a+b'))
finally:
os.umask(omask)

Expand All @@ -162,9 +162,11 @@ def __archive_to_mbox(self, post):
"""Retain a text copy of the message in an mbox file."""
try:
afn = self.ArchiveFileName()
syslog('debug', 'Archiver: Writing to mbox file: %s', afn)
mbox = self.__archive_file(afn)
mbox.AppendMessage(post)
mbox.fp.close()
mbox.close()
syslog('debug', 'Archiver: Successfully wrote message to mbox file: %s', afn)
except IOError as msg:
syslog('error', 'Archive file access failure:\n\t%s %s', afn, msg)
raise
Expand All @@ -186,32 +188,56 @@ def ExternalArchive(self, ar, txt):
#
def ArchiveMail(self, msg):
"""Store postings in mbox and/or pipermail archive, depending."""
from Mailman.Logging.Syslog import syslog
syslog('debug', 'Archiver: Starting ArchiveMail for list %s', self.internal_name())

# Fork so archival errors won't disrupt normal list delivery
if mm_cfg.ARCHIVE_TO_MBOX == -1:
syslog('debug', 'Archiver: ARCHIVE_TO_MBOX is -1, archiving disabled')
return

syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = %s', mm_cfg.ARCHIVE_TO_MBOX)
#
# We don't need an extra archiver lock here because we know the list
# itself must be locked.
if mm_cfg.ARCHIVE_TO_MBOX in (1, 2):
syslog('debug', 'Archiver: Writing to mbox archive')
self.__archive_to_mbox(msg)
if mm_cfg.ARCHIVE_TO_MBOX == 1:
# Archive to mbox only.
syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = 1, mbox only, returning')
return
txt = str(msg)

txt = msg.as_string()
unixfrom = msg.get_unixfrom()
# Handle case where unixfrom is None (Python 3 compatibility)
if unixfrom and not txt.startswith(unixfrom):
txt = unixfrom + '\n' + txt

# should we use the internal or external archiver?
private_p = self.archive_private
syslog('debug', 'Archiver: archive_private = %s', private_p)

if mm_cfg.PUBLIC_EXTERNAL_ARCHIVER and not private_p:
syslog('debug', 'Archiver: Using public external archiver')
self.ExternalArchive(mm_cfg.PUBLIC_EXTERNAL_ARCHIVER, txt)
elif mm_cfg.PRIVATE_EXTERNAL_ARCHIVER and private_p:
syslog('debug', 'Archiver: Using private external archiver')
self.ExternalArchive(mm_cfg.PRIVATE_EXTERNAL_ARCHIVER, txt)
else:
# use the internal archiver
f = StringIO(txt)
syslog('debug', 'Archiver: Using internal HyperArch archiver')
f = tempfile.NamedTemporaryFile()
if isinstance(txt, str):
txt = txt.encode('utf-8')
f.write(txt)
f.flush()
from . import HyperArch
h = HyperArch.HyperArchive(self)
h.processUnixMailbox(f)
h.close()
f.close()
syslog('debug', 'Archiver: Completed internal archiving')

#
# called from MailList.MailList.Save()
Expand Down
56 changes: 32 additions & 24 deletions Mailman/Archiver/HyperArch.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from email.header import decode_header, make_header
from email.errors import HeaderParseError
from email.charset import Charset
from functools import cmp_to_key

from Mailman import mm_cfg
from Mailman import Utils
Expand Down Expand Up @@ -94,7 +95,7 @@ def html_quote(s, lang=None):
('"', '"'))
for thing, repl in repls:
s = s.replace(thing, repl)
return Utils.uncanonstr(s, lang)
return s


def url_quote(s):
Expand Down Expand Up @@ -136,7 +137,7 @@ def CGIescape(arg, lang=None):
s = Utils.websafe(arg)
else:
s = Utils.websafe(str(arg))
return Utils.uncanonstr(s.replace('"', '"'), lang)
return s.replace('"', '"')

# Parenthesized human name
paren_name_pat = re.compile(r'([(].*[)])')
Expand Down Expand Up @@ -223,7 +224,7 @@ def quick_maketext(templatefile, dict=None, lang=None, mlist=None):
syslog('error', 'broken template: %s\n%s', filepath, e)
# Make sure the text is in the given character set, or html-ify any bogus
# characters.
return Utils.uncanonstr(text, lang)
return text



Expand Down Expand Up @@ -298,7 +299,7 @@ def __init__(self, message=None, sequence=0, keepHeaders=[],
cset_out = Charset(cset).output_charset or cset
if isinstance(cset_out, str):
# email 3.0.1 (python 2.4) doesn't like unicode
cset_out = cset_out.encode('us-ascii')
cset_out = cset_out.encode('us-ascii', 'replace')
charset = message.get_content_charset(cset_out)
if charset:
charset = charset.lower().strip()
Expand All @@ -311,12 +312,17 @@ def __init__(self, message=None, sequence=0, keepHeaders=[],
except binascii.Error:
body = None
if body and charset != Utils.GetCharSet(self._lang):
if isinstance(charset, bytes):
charset = charset.decode('utf-8', 'replace')
# decode body
try:
body = str(body, charset)
body = body.decode(charset)
except (UnicodeError, LookupError):
body = None
if body:
# Handle both bytes and strings properly
if isinstance(body, bytes):
body = body.decode('utf-8', 'replace')
self.body = [l + "\n" for l in body.splitlines()]

self.decode_headers()
Expand Down Expand Up @@ -414,9 +420,9 @@ def decode_headers(self):
otrans = i18n.get_translation()
try:
i18n.set_language(self._lang)
atmark = str(_(' at '), Utils.GetCharSet(self._lang))
atmark = _(' at ')
subject = re.sub(r'([-+,.\w]+)@([-+.\w]+)',
'\g<1>' + atmark + '\g<2>', subject)
r'\g<1>' + atmark + r'\g<2>', subject)
finally:
i18n.set_translation(otrans)
self.decoded['subject'] = subject
Expand All @@ -429,22 +435,22 @@ def strip_subject(self, subject):
if prefix:
prefix_pat = re.escape(prefix)
prefix_pat = '%'.join(prefix_pat.split(r'\%'))
prefix_pat = re.sub(r'%\d*d', r'\s*\d+\s*', prefix_pat)
prefix_pat = re.sub(r'%\d*d', r'\\\\s*\\\\d+\\\\s*', prefix_pat)
subject = re.sub(prefix_pat, '', subject)
subject = subject.lstrip()
# MAS Should we strip FW and FWD too?
strip_pat = re.compile('^((RE|AW|SV|VS)(\[\d+\])?:\s*)+', re.I)
strip_pat = re.compile(r'^((RE|AW|SV|VS)(\[\d+\])?:\s*)+', re.I)
stripped = strip_pat.sub('', subject)
# Also remove whitespace to avoid folding/unfolding differences
stripped = re.sub('\s', '', stripped)
stripped = re.sub(r'\s', '', stripped)
return stripped

def decode_charset(self, field):
# TK: This function was rewritten for unifying to Unicode.
# Convert 'field' into Unicode one line string.
try:
pairs = decode_header(field)
ustr = make_header(pairs).__unicode__()
ustr = make_header(pairs).__str__()
except (LookupError, UnicodeError, ValueError, HeaderParseError):
# assume list's language
cset = Utils.GetCharSet(self._mlist.preferred_language)
Expand Down Expand Up @@ -519,8 +525,8 @@ def _get_subject_enc(self, art):

def _get_next(self):
"""Return the href and subject for the previous message"""
if self.__next__:
subject = self._get_subject_enc(self.__next__)
if hasattr( self, 'next' ) and self.next is not None:
subject = self._get_subject_enc(self.next)
next = ('<LINK REL="Next" HREF="%s">'
% (url_quote(self.next.filename)))
next_wsubj = ('<LI>' + _('Next message (by thread):') +
Expand All @@ -540,6 +546,7 @@ def _get_body(self):
body = self.html_body
except AttributeError:
body = self.body

return null_to_space(EMPTYSTRING.join(body))

def _add_decoded(self, d):
Expand Down Expand Up @@ -581,13 +588,14 @@ def as_text(self):
otrans = i18n.get_translation()
try:
i18n.set_language(self._lang)
atmark = str(_(' at '), cset)
atmark = _(' at ')
if isinstance(atmark, bytes):
atmark = str(atmark, cset)
body = re.sub(r'([-+,.\w]+)@([-+.\w]+)',
'\g<1>' + atmark + '\g<2>', body)
r'\g<1>' + atmark + r'\g<2>', body)
finally:
i18n.set_translation(otrans)
# Return body to character set of article.
body = body.encode(cset, 'replace')

return NL.join(headers) % d + '\n\n' + body + '\n'

def _set_date(self, message):
Expand Down Expand Up @@ -870,7 +878,7 @@ def processListArch(self):
#if the working file is still here, the archiver may have
# crashed during archiving. Save it, log an error, and move on.
try:
wf = open(wname)
wf = open(wname, 'r')
syslog('error',
'Archive working file %s present. '
'Check %s for possibly unarchived msgs',
Expand All @@ -890,7 +898,7 @@ def processListArch(self):
except IOError:
pass
os.rename(name,wname)
archfile = open(wname)
archfile = open(wname, 'r')
self.processUnixMailbox(archfile)
archfile.close()
os.unlink(wname)
Expand Down Expand Up @@ -1017,7 +1025,7 @@ def sf(a, b):
else:
return 0
if self.ARCHIVE_PERIOD in ('month','year','quarter'):
self.archives.sort(sf)
self.archives.sort(key = cmp_to_key(sf))
else:
self.archives.sort()
self.archives.reverse()
Expand All @@ -1034,7 +1042,7 @@ def open_new_archive(self, archive, archivedir):
index_html = os.path.join(archivedir, 'index.html')
try:
os.unlink(index_html)
except:
except (OSError, IOError):
pass
os.symlink(self.DEFAULTINDEX+'.html',index_html)

Expand Down Expand Up @@ -1139,7 +1147,7 @@ def update_archive(self, archive):
oldgzip = os.path.join(self.basedir, '%s.old.txt.gz' % archive)
try:
# open the plain text file
archt = open(txtfile)
archt = open(txtfile, 'r')
except IOError:
return
try:
Expand Down Expand Up @@ -1185,8 +1193,6 @@ def __processbody_URLquote(self, lines):
# 3. make it faster
# TK: Prepare for unicode obscure.
atmark = _(' at ')
if lines and isinstance(lines[0], str):
atmark = str(atmark, Utils.GetCharSet(self.lang), 'replace')
source = lines[:]
dest = lines
last_line_was_quoted = 0
Expand Down Expand Up @@ -1250,6 +1256,8 @@ def __processbody_URLquote(self, lines):
kr = urlpat.search(L)
if jr is None and kr is None:
L = CGIescape(L, self.lang)
if isinstance(L, bytes):
L = L.decode('utf-8')
L = prefix + L2 + L + suffix
source[i] = None
dest[i] = L
Expand Down
Loading