Skip to content

Commit 62b598b

Browse files
committed
Update pyarchivefile.py
1 parent 5c603a4 commit 62b598b

1 file changed

Lines changed: 84 additions & 7 deletions

File tree

pyarchivefile.py

Lines changed: 84 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,17 @@
7373
except ImportError:
7474
import json
7575

76+
testyaml = False
77+
try:
78+
import oyaml as yaml
79+
testyaml = True
80+
except ImportError:
81+
try:
82+
import yaml
83+
testyaml = True
84+
except ImportError:
85+
testyaml = False
86+
7687
try:
7788
import configparser
7889
except ImportError:
@@ -2612,7 +2623,7 @@ def _load_all_members_spooled(self):
26122623
scanned_leading = 0 # for tolerant header scan
26132624

26142625
while True:
2615-
data = self.file.read(1 << 20) # 1 MiB blocks
2626+
data = self.file.read(__filebuff_size__) # 1 MiB blocks
26162627
if not data:
26172628
if d is not None:
26182629
self._spool.write(d.flush())
@@ -2770,7 +2781,7 @@ def write(self, data):
27702781

27712782
# Buffer and compress in chunks to limit memory
27722783
self._write_buf += data
2773-
if len(self._write_buf) >= (1 << 20): # 1 MiB threshold
2784+
if len(self._write_buf) >= (__filebuff_size__): # 1 MiB threshold
27742785
chunk = self._compressor.compress(bytes(self._write_buf))
27752786
if chunk:
27762787
self.file.write(chunk)
@@ -3075,7 +3086,7 @@ def _load_all_members_spooled(self):
30753086

30763087
self._spool = tempfile.SpooledTemporaryFile(max_size=self.spool_threshold)
30773088

3078-
CHUNK = 1 << 20
3089+
CHUNK = __filebuff_size__
30793090
pending = b""
30803091
d = None
30813092
absolute_offset = 0
@@ -3238,7 +3249,7 @@ def write(self, data):
32383249

32393250
# Stage and compress in chunks
32403251
self._write_buf += data
3241-
if len(self._write_buf) >= (1 << 20): # 1 MiB threshold
3252+
if len(self._write_buf) >= (__filebuff_size__): # 1 MiB threshold
32423253
out = self._compressor.compress(bytes(self._write_buf))
32433254
if out:
32443255
self.file.write(out)
@@ -3691,7 +3702,7 @@ def GetFileChecksum(inbytes, checksumtype="md5", encodedata=True, formatspecs=__
36913702
if CheckSumSupport(algo_key, hashlib_guaranteed):
36923703
h = hashlib.new(algo_key)
36933704
while True:
3694-
chunk = inbytes.read(1 << 20)
3705+
chunk = inbytes.read(__filebuff_size__)
36953706
if not chunk:
36963707
break
36973708
if not isinstance(chunk, (bytes, bytearray, memoryview)):
@@ -4143,6 +4154,28 @@ def ReadFileHeaderDataWithContent(fp, listonly=False, uncompress=True, skipcheck
41434154
fprejsoncontent = ""
41444155
fjsonrawcontent = fprejsoncontent
41454156
fjsoncontent = {}
4157+
elif(testyaml and fjsontype == "yaml"):
4158+
fjsoncontent = {}
4159+
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
4160+
if (fjsonsize > 0):
4161+
try:
4162+
# try base64 → utf-8 → YAML
4163+
fjsonrawcontent = base64.b64decode(fprejsoncontent.encode("UTF-8")).decode("UTF-8")
4164+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4165+
except (binascii.Error, UnicodeDecodeError, yaml.YAMLError):
4166+
try:
4167+
# fall back to treating the bytes as plain text YAML
4168+
fjsonrawcontent = fprejsoncontent
4169+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4170+
except (UnicodeDecodeError, yaml.YAMLError):
4171+
# final fallback: empty
4172+
fprejsoncontent = ""
4173+
fjsonrawcontent = fprejsoncontent
4174+
fjsoncontent = {}
4175+
else:
4176+
fprejsoncontent = ""
4177+
fjsonrawcontent = fprejsoncontent
4178+
fjsoncontent = {}
41464179
elif(fjsontype=="list"):
41474180
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
41484181
flisttmp = MkTempFile()
@@ -4316,6 +4349,28 @@ def ReadFileHeaderDataWithContentToArray(fp, listonly=False, contentasfile=True,
43164349
fprejsoncontent = ""
43174350
fjsonrawcontent = fprejsoncontent
43184351
fjsoncontent = {}
4352+
elif(testyaml and fjsontype == "yaml"):
4353+
fjsoncontent = {}
4354+
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
4355+
if (fjsonsize > 0):
4356+
try:
4357+
# try base64 → utf-8 → YAML
4358+
fjsonrawcontent = base64.b64decode(fprejsoncontent.encode("UTF-8")).decode("UTF-8")
4359+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4360+
except (binascii.Error, UnicodeDecodeError, yaml.YAMLError):
4361+
try:
4362+
# fall back to treating the bytes as plain text YAML
4363+
fjsonrawcontent = fprejsoncontent
4364+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4365+
except (UnicodeDecodeError, yaml.YAMLError):
4366+
# final fallback: empty
4367+
fprejsoncontent = ""
4368+
fjsonrawcontent = fprejsoncontent
4369+
fjsoncontent = {}
4370+
else:
4371+
fprejsoncontent = ""
4372+
fjsonrawcontent = fprejsoncontent
4373+
fjsoncontent = {}
43194374
elif(fjsontype=="list"):
43204375
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
43214376
flisttmp = MkTempFile()
@@ -4502,6 +4557,28 @@ def ReadFileHeaderDataWithContentToList(fp, listonly=False, contentasfile=False,
45024557
fprejsoncontent = ""
45034558
fjsonrawcontent = fprejsoncontent
45044559
fjsoncontent = {}
4560+
elif(testyaml and fjsontype == "yaml"):
4561+
fjsoncontent = {}
4562+
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
4563+
if (fjsonsize > 0):
4564+
try:
4565+
# try base64 → utf-8 → YAML
4566+
fjsonrawcontent = base64.b64decode(fprejsoncontent.encode("UTF-8")).decode("UTF-8")
4567+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4568+
except (binascii.Error, UnicodeDecodeError, yaml.YAMLError):
4569+
try:
4570+
# fall back to treating the bytes as plain text YAML
4571+
fjsonrawcontent = fprejsoncontent
4572+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4573+
except (UnicodeDecodeError, yaml.YAMLError):
4574+
# final fallback: empty
4575+
fprejsoncontent = ""
4576+
fjsonrawcontent = fprejsoncontent
4577+
fjsoncontent = {}
4578+
else:
4579+
fprejsoncontent = ""
4580+
fjsonrawcontent = fprejsoncontent
4581+
fjsoncontent = {}
45054582
elif(fjsontype=="list"):
45064583
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
45074584
flisttmp = MkTempFile()
@@ -8758,7 +8835,7 @@ def ensure_filelike(infile, mode="rb", use_mmap=False, **adapter_kw):
87588835

87598836
# ========= copy helpers =========
87608837

8761-
def fast_copy(infp, outfp, bufsize=1 << 20):
8838+
def fast_copy(infp, outfp, bufsize=__filebuff_size__):
87628839
"""
87638840
Efficient copy from any readable file-like to any writable file-like.
87648841
Uses readinto() when available to avoid extra allocations.
@@ -8802,7 +8879,7 @@ def copy_file_to_mmap_dest(src_path, outfp, chunk_size=__spoolfile_size__):
88028879
shutil.copyfileobj(fp, outfp, length=chunk_size)
88038880

88048881

8805-
def copy_opaque(src, dst, bufsize=1 << 20, grow_step=64 << 20):
8882+
def copy_opaque(src, dst, bufsize=__filebuff_size__, grow_step=64 << 20):
88068883
"""
88078884
Copy opaque bytes from 'src' (any readable file-like) to 'dst'
88088885
(your mmap-backed FileLikeAdapter or any writable file-like).

0 commit comments

Comments
 (0)