Changes to fix cgi and portability with respect to the python interpreter location

.gitignore

@@ -0,0 +1,86 @@
+mailman.po~
+Mailman/Archiver/Makefile
+Mailman/Bouncers/Makefile
+Mailman/Cgi/Makefile
+Mailman/Commands/Makefile
+Mailman/Defaults.py
+Mailman/Gui/Makefile
+Mailman/Handlers/Makefile
+Mailman/Logging/Makefile
+Mailman/MTA/Makefile
+Mailman/Makefile
+Mailman/Queue/Makefile
+Mailman/__pycache__/
+Mailman/mm_cfg.py.dist
+Makefile
+bin/Makefile
+build/
+config.log
+config.status
+cron/Makefile
+cron/crontab.in
+messages/Makefile
+messages/ar/LC_MESSAGES/mailman.mo
+messages/ast/LC_MESSAGES/mailman.mo
+messages/ca/LC_MESSAGES/mailman.mo
+messages/cs/LC_MESSAGES/mailman.mo
+messages/da/LC_MESSAGES/mailman.mo
+messages/de/LC_MESSAGES/mailman.mo
+messages/el/LC_MESSAGES/mailman.mo
+messages/eo/LC_MESSAGES/mailman.mo
+messages/es/LC_MESSAGES/mailman.mo
+messages/et/LC_MESSAGES/mailman.mo
+messages/eu/LC_MESSAGES/mailman.mo
+messages/fa/LC_MESSAGES/mailman.mo
+messages/fi/LC_MESSAGES/mailman.mo
+messages/fr/LC_MESSAGES/mailman.mo
+messages/gl/LC_MESSAGES/mailman.mo
+messages/he/LC_MESSAGES/mailman.mo
+messages/hr/LC_MESSAGES/mailman.mo
+messages/hu/LC_MESSAGES/mailman.mo
+messages/ia/LC_MESSAGES/mailman.mo
+messages/it/LC_MESSAGES/mailman.mo
+messages/ja/LC_MESSAGES/mailman.mo
+messages/ko/LC_MESSAGES/mailman.mo
+messages/lt/LC_MESSAGES/mailman.mo
+messages/nl/LC_MESSAGES/mailman.mo
+messages/no/LC_MESSAGES/mailman.mo
+messages/pl/LC_MESSAGES/mailman.mo
+messages/pt/LC_MESSAGES/mailman.mo
+messages/pt_BR/LC_MESSAGES/mailman.mo
+messages/ro/LC_MESSAGES/mailman.mo
+messages/ru/LC_MESSAGES/mailman.mo
+messages/sk/LC_MESSAGES/mailman.mo
+messages/sl/LC_MESSAGES/mailman.mo
+messages/sr/LC_MESSAGES/mailman.mo
+messages/sv/LC_MESSAGES/mailman.mo
+messages/tr/LC_MESSAGES/mailman.mo
+messages/uk/LC_MESSAGES/mailman.mo
+messages/vi/LC_MESSAGES/mailman.mo
+messages/zh_CN/LC_MESSAGES/mailman.mo
+messages/zh_TW/LC_MESSAGES/mailman.mo
+misc/Makefile
+misc/mailman
+misc/paths.py
+scripts/Makefile
+src/Makefile
+src/admin
+src/admindb
+src/common.o
+src/confirm
+src/create
+src/edithtml
+src/listinfo
+src/mailman
+src/options
+src/private
+src/rmlist
+src/roster
+src/subscribe
+src/vsnprintf.o
+templates/Makefile
+templates/.converted.stamp
+tests/Makefile
+tests/bounces/Makefile
+tests/msgs/Makefile
+messages/.converted.stamp

Mailman/Archiver/Archiver.py

@@ -29,7 +29,7 @@
 import errno
 import traceback
 import re
-from io import StringIO
+import tempfile
 
 from Mailman import mm_cfg
 from Mailman import Mailbox
@@ -150,7 +150,7 @@ def __archive_file(self, afn):
         """Open (creating, if necessary) the named archive file."""
         omask = os.umask(0o002)
         try:
-            return Mailbox.Mailbox(open(afn, 'a+'))
+            return Mailbox.Mailbox(open(afn, 'a+b'))
         finally:
             os.umask(omask)
 
@@ -162,9 +162,11 @@ def __archive_to_mbox(self, post):
         """Retain a text copy of the message in an mbox file."""
         try:
             afn = self.ArchiveFileName()
+            syslog('debug', 'Archiver: Writing to mbox file: %s', afn)
             mbox = self.__archive_file(afn)
             mbox.AppendMessage(post)
-            mbox.fp.close()
+            mbox.close()
+            syslog('debug', 'Archiver: Successfully wrote message to mbox file: %s', afn)
         except IOError as msg:
             syslog('error', 'Archive file access failure:\n\t%s %s', afn, msg)
             raise
@@ -186,32 +188,56 @@ def ExternalArchive(self, ar, txt):
     #
     def ArchiveMail(self, msg):
         """Store postings in mbox and/or pipermail archive, depending."""
+        from Mailman.Logging.Syslog import syslog
+        syslog('debug', 'Archiver: Starting ArchiveMail for list %s', self.internal_name())
+
         # Fork so archival errors won't disrupt normal list delivery
         if mm_cfg.ARCHIVE_TO_MBOX == -1:
+            syslog('debug', 'Archiver: ARCHIVE_TO_MBOX is -1, archiving disabled')
             return
+
+        syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = %s', mm_cfg.ARCHIVE_TO_MBOX)
         #
         # We don't need an extra archiver lock here because we know the list
         # itself must be locked.
         if mm_cfg.ARCHIVE_TO_MBOX in (1, 2):
+            syslog('debug', 'Archiver: Writing to mbox archive')
             self.__archive_to_mbox(msg)
             if mm_cfg.ARCHIVE_TO_MBOX == 1:
                 # Archive to mbox only.
+                syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = 1, mbox only, returning')
                 return
-        txt = str(msg)
+
+        txt = msg.as_string()
+        unixfrom = msg.get_unixfrom()
+        # Handle case where unixfrom is None (Python 3 compatibility)
+        if unixfrom and not txt.startswith(unixfrom):
+            txt = unixfrom + '\n' + txt
+
         # should we use the internal or external archiver?
         private_p = self.archive_private
+        syslog('debug', 'Archiver: archive_private = %s', private_p)
+
         if mm_cfg.PUBLIC_EXTERNAL_ARCHIVER and not private_p:
+            syslog('debug', 'Archiver: Using public external archiver')
             self.ExternalArchive(mm_cfg.PUBLIC_EXTERNAL_ARCHIVER, txt)
         elif mm_cfg.PRIVATE_EXTERNAL_ARCHIVER and private_p:
+            syslog('debug', 'Archiver: Using private external archiver')
             self.ExternalArchive(mm_cfg.PRIVATE_EXTERNAL_ARCHIVER, txt)
         else:
             # use the internal archiver
-            f = StringIO(txt)
+            syslog('debug', 'Archiver: Using internal HyperArch archiver')
+            f = tempfile.NamedTemporaryFile()
+            if isinstance(txt, str):
+                txt = txt.encode('utf-8')
+            f.write(txt)
+            f.flush()
             from . import HyperArch
             h = HyperArch.HyperArchive(self)
             h.processUnixMailbox(f)
             h.close()
             f.close()
+            syslog('debug', 'Archiver: Completed internal archiving')
 
     #
     # called from MailList.MailList.Save()

Mailman/Archiver/HyperArch.py

@@ -41,6 +41,7 @@
 from email.header import decode_header, make_header
 from email.errors import HeaderParseError
 from email.charset import Charset
+from functools import cmp_to_key
 
 from Mailman import mm_cfg
 from Mailman import Utils
@@ -94,7 +95,7 @@ def html_quote(s, lang=None):
               ('"', '"'))
     for thing, repl in repls:
         s = s.replace(thing, repl)
-    return Utils.uncanonstr(s, lang)
+    return s
 
 
 def url_quote(s):
@@ -136,7 +137,7 @@ def CGIescape(arg, lang=None):
         s = Utils.websafe(arg)
     else:
         s = Utils.websafe(str(arg))
-    return Utils.uncanonstr(s.replace('"', '"'), lang)
+    return s.replace('"', '"')
 
 # Parenthesized human name
 paren_name_pat = re.compile(r'([(].*[)])')
@@ -223,7 +224,7 @@ def quick_maketext(templatefile, dict=None, lang=None, mlist=None):
             syslog('error', 'broken template: %s\n%s', filepath, e)
     # Make sure the text is in the given character set, or html-ify any bogus
     # characters.
-    return Utils.uncanonstr(text, lang)
+    return text
 
 
 
@@ -298,7 +299,7 @@ def __init__(self, message=None, sequence=0, keepHeaders=[],
         cset_out = Charset(cset).output_charset or cset
         if isinstance(cset_out, str):
             # email 3.0.1 (python 2.4) doesn't like unicode
-            cset_out = cset_out.encode('us-ascii')
+            cset_out = cset_out.encode('us-ascii', 'replace')
         charset = message.get_content_charset(cset_out)
         if charset:
             charset = charset.lower().strip()
@@ -311,12 +312,17 @@ def __init__(self, message=None, sequence=0, keepHeaders=[],
             except binascii.Error:
                 body = None
             if body and charset != Utils.GetCharSet(self._lang):
+                if isinstance(charset, bytes):
+                    charset = charset.decode('utf-8', 'replace')
                 # decode body
                 try:
-                    body = str(body, charset)
+                    body = body.decode(charset)
                 except (UnicodeError, LookupError):
                     body = None
             if body:
+                # Handle both bytes and strings properly
+                if isinstance(body, bytes):
+                    body = body.decode('utf-8', 'replace')
                 self.body = [l + "\n" for l in body.splitlines()]
 
         self.decode_headers()
@@ -414,9 +420,9 @@ def decode_headers(self):
                 otrans = i18n.get_translation()
                 try:
                     i18n.set_language(self._lang)
-                    atmark = str(_(' at '), Utils.GetCharSet(self._lang))
+                    atmark = _(' at ')
                     subject = re.sub(r'([-+,.\w]+)@([-+.\w]+)',
-                              '\g<1>' + atmark + '\g<2>', subject)
+                              r'\g<1>' + atmark + r'\g<2>', subject)
                 finally:
                     i18n.set_translation(otrans)
             self.decoded['subject'] = subject
@@ -429,22 +435,22 @@ def strip_subject(self, subject):
         if prefix:
             prefix_pat = re.escape(prefix)
             prefix_pat = '%'.join(prefix_pat.split(r'\%'))
-            prefix_pat = re.sub(r'%\d*d', r'\s*\d+\s*', prefix_pat)
+            prefix_pat = re.sub(r'%\d*d', r'\\\\s*\\\\d+\\\\s*', prefix_pat)
             subject = re.sub(prefix_pat, '', subject)
         subject = subject.lstrip()
         # MAS Should we strip FW and FWD too?
-        strip_pat = re.compile('^((RE|AW|SV|VS)(\[\d+\])?:\s*)+', re.I)
+        strip_pat = re.compile(r'^((RE|AW|SV|VS)(\[\d+\])?:\s*)+', re.I)
         stripped = strip_pat.sub('', subject)
         # Also remove whitespace to avoid folding/unfolding differences
-        stripped = re.sub('\s', '', stripped)
+        stripped = re.sub(r'\s', '', stripped)
         return stripped
 
     def decode_charset(self, field):
         # TK: This function was rewritten for unifying to Unicode.
         # Convert 'field' into Unicode one line string.
         try:
             pairs = decode_header(field)
-            ustr = make_header(pairs).__unicode__()
+            ustr = make_header(pairs).__str__()
         except (LookupError, UnicodeError, ValueError, HeaderParseError):
             # assume list's language
             cset = Utils.GetCharSet(self._mlist.preferred_language)
@@ -519,8 +525,8 @@ def _get_subject_enc(self, art):
 
     def _get_next(self):
         """Return the href and subject for the previous message"""
-        if self.__next__:
-            subject = self._get_subject_enc(self.__next__)
+        if hasattr( self, 'next' ) and self.next is not None:
+            subject = self._get_subject_enc(self.next)
             next = ('<LINK REL="Next"  HREF="%s">'
                     % (url_quote(self.next.filename)))
             next_wsubj = ('<LI>' + _('Next message (by thread):') +
@@ -540,6 +546,7 @@ def _get_body(self):
             body = self.html_body
         except AttributeError:
             body = self.body
+
         return null_to_space(EMPTYSTRING.join(body))
 
     def _add_decoded(self, d):
@@ -581,13 +588,14 @@ def as_text(self):
             otrans = i18n.get_translation()
             try:
                 i18n.set_language(self._lang)
-                atmark = str(_(' at '), cset)
+                atmark = _(' at ')
+                if isinstance(atmark, bytes):
+                    atmark = str(atmark, cset)
                 body = re.sub(r'([-+,.\w]+)@([-+.\w]+)',
-                              '\g<1>' + atmark + '\g<2>', body)
+                              r'\g<1>' + atmark + r'\g<2>', body)
             finally:
                 i18n.set_translation(otrans)
-        # Return body to character set of article.
-        body = body.encode(cset, 'replace')
+
         return NL.join(headers) % d + '\n\n' + body + '\n'
 
     def _set_date(self, message):
@@ -870,7 +878,7 @@ def processListArch(self):
         #if the working file is still here, the archiver may have
         # crashed during archiving. Save it, log an error, and move on.
         try:
-            wf = open(wname)
+            wf = open(wname, 'r')
             syslog('error',
                    'Archive working file %s present.  '
                    'Check %s for possibly unarchived msgs',
@@ -890,7 +898,7 @@ def processListArch(self):
         except IOError:
             pass
         os.rename(name,wname)
-        archfile = open(wname)
+        archfile = open(wname, 'r')
         self.processUnixMailbox(archfile)
         archfile.close()
         os.unlink(wname)
@@ -1017,7 +1025,7 @@ def sf(a, b):
             else:
                 return 0
         if self.ARCHIVE_PERIOD in ('month','year','quarter'):
-            self.archives.sort(sf)
+            self.archives.sort(key = cmp_to_key(sf))
         else:
             self.archives.sort()
         self.archives.reverse()
@@ -1034,7 +1042,7 @@ def open_new_archive(self, archive, archivedir):
         index_html = os.path.join(archivedir, 'index.html')
         try:
             os.unlink(index_html)
-        except:
+        except (OSError, IOError):
             pass
         os.symlink(self.DEFAULTINDEX+'.html',index_html)
 
@@ -1139,7 +1147,7 @@ def update_archive(self, archive):
             oldgzip = os.path.join(self.basedir, '%s.old.txt.gz' % archive)
             try:
                 # open the plain text file
-                archt = open(txtfile)
+                archt = open(txtfile, 'r')
             except IOError:
                 return
             try:
@@ -1185,8 +1193,6 @@ def __processbody_URLquote(self, lines):
         # 3. make it faster
         # TK: Prepare for unicode obscure.
         atmark = _(' at ')
-        if lines and isinstance(lines[0], str):
-            atmark = str(atmark, Utils.GetCharSet(self.lang), 'replace')
         source = lines[:]
         dest = lines
         last_line_was_quoted = 0
@@ -1250,6 +1256,8 @@ def __processbody_URLquote(self, lines):
                 kr = urlpat.search(L)
             if jr is None and kr is None:
                 L = CGIescape(L, self.lang)
+            if isinstance(L, bytes):
+                L = L.decode('utf-8')
             L = prefix + L2 + L + suffix
             source[i] = None
             dest[i] = L