Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Doc/whatsnew/3.16.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1045,6 +1045,16 @@ that may require changes to your code.
raises :exc:`io.UnsupportedOperation` unless buffering is disabled.
(Contributed by An Long in :gh:`86768`.)

* :mod:`gzip` and :mod:`tarfile` now derive the ``FNAME`` field of the gzip
header from the file name like :program:`gunzip` does: the suffix is matched
ignoring case, and ``.tgz`` is replaced with ``.tar`` instead of being left
in place. For example, an archive created as :file:`spam.tgz` now records
``spam.tar`` rather than ``spam.tgz``, and :file:`spam.GZ` records ``spam``
rather than ``spam.GZ``. Code comparing generated files byte for byte may
need to be updated. ``python -m gzip -d`` accepts the same names, where it
previously refused anything not ending in a lowercase ``.gz``.
(Contributed by Dmitry Voropaev in :gh:`88661`.)


Build changes
=============
Expand Down
31 changes: 26 additions & 5 deletions Lib/gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,25 @@ def writable(self):
return True


def _gunzip_name(name):
"""The name :program:`gunzip` would decompress *name* to, or ``None``.

The suffix is matched ignoring case, and ``.tgz`` becomes ``.tar`` rather
than being stripped. ``.taz`` is not handled: it means a ``.tar.Z``, and
this module does not do :program:`compress`. Accepts and returns either
:class:`str` or :class:`bytes`.
"""
if isinstance(name, bytes):
gz, tgz, tar = b'.gz', b'.tgz', b'.tar'
else:
gz, tgz, tar = '.gz', '.tgz', '.tar'
if name[-3:].lower() == gz:
return name[:-3]
if name[-4:].lower() == tgz:
return name[:-4] + tar
return None


class GzipFile(_streams.BaseStream):
"""The GzipFile class simulates most of the methods of a file object with
the exception of the truncate() method.
Expand Down Expand Up @@ -288,8 +307,9 @@ def _write_gzip_header(self, compresslevel):
fname = os.path.basename(self.name)
if not isinstance(fname, bytes):
fname = fname.encode('latin-1')
if fname.endswith(b'.gz'):
fname = fname[:-3]
stripped = _gunzip_name(fname)
if stripped is not None:
fname = stripped
except UnicodeEncodeError:
fname = b''
flags = 0
Expand Down Expand Up @@ -729,10 +749,11 @@ def main():
f = GzipFile(filename="", mode="rb", fileobj=sys.stdin.buffer)
g = sys.stdout.buffer
else:
if arg[-3:] != ".gz":
sys.exit(f"filename doesn't end in .gz: {arg!r}")
out = _gunzip_name(arg)
if out is None:
sys.exit(f"filename doesn't end in .gz or .tgz: {arg!r}")
f = open(arg, "rb")
g = builtins.open(arg[:-3], "wb")
g = builtins.open(out, "wb")
else:
if arg == "-":
f = sys.stdin.buffer
Expand Down
7 changes: 6 additions & 1 deletion Lib/tarfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,8 +456,13 @@ def _init_write_gz(self, compresslevel, mtime):
mtime = int(time.time())
timestamp = struct.pack("<L", mtime)
self.__write(b"\037\213\010\010" + timestamp + b"\002\377")
if self.name.endswith(".gz"):
# Like gunzip, match the suffix ignoring case, and turn ".tgz" into
# ".tar" instead of just stripping it. (gzip._gunzip_name does the
# same, but importing gzip here would pull it into the w|gz path.)
if self.name[-3:].lower() == ".gz":
self.name = self.name[:-3]
elif self.name[-4:].lower() == ".tgz":
self.name = self.name[:-4] + ".tar"
# Honor "directory components removed" from RFC1952
self.name = os.path.basename(self.name)
# RFC1952 says we must use ISO-8859-1 for the FNAME field.
Expand Down
68 changes: 64 additions & 4 deletions Lib/test/test_gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,39 @@ def test_metadata_ascii_name(self):
self.filename = os_helper.TESTFN_ASCII
self.test_metadata()

def test_metadata_name_suffix(self):
# gh-88661: the FNAME field holds the name that the file is expected
# to have after decompression. Like gunzip, the suffix is matched
# ignoring case, and ".tgz" is turned into ".tar".
base = os_helper.TESTFN_ASCII
# Only the suffix is matched ignoring case; the rest of the name
# keeps the case it was given, the way make_ofname() does in gunzip.
upper = base.upper()
for filename, expected in ((base, base),
(base + '.gz', base),
(base + '.GZ', base),
(base + '.gZ', base),
(base + '.tgz', base + '.tar'),
(base + '.TGZ', base + '.tar'),
(base + '.tGz', base + '.tar'),
(base + '.tar', base + '.tar'),
# .taz means .tar.Z, which this module cannot read.
(base + '.taz', base + '.taz'),
(base + '.tgz.gz', base + '.tgz'),
(upper + '.GZ', upper),
(upper + '.TGZ', upper + '.tar')):
with self.subTest(filename=filename):
try:
with gzip.GzipFile(filename, 'w') as f:
f.write(data1)
with open(filename, 'rb') as f:
header = f.read(1024)
self.assertEqual(header[3], 8) # only the FNAME flag
fname = header[10:header.index(b'\0', 10)]
self.assertEqual(fname.decode('latin-1'), expected)
finally:
os_helper.unlink(filename)

def test_compresslevel_metadata(self):
# see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
# specifically, discussion of XFL in section 2.3.1
Expand Down Expand Up @@ -1153,10 +1186,37 @@ def test_decompress_infile_outfile(self):
self.assertEqual(err, b'')

def test_decompress_infile_outfile_error(self):
rc, out, err = assert_python_failure('-m', 'gzip', '-d', 'thisisatest.out')
self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'", err.strip())
self.assertEqual(rc, 1)
self.assertEqual(out, b'')
# ".taz" is a ".tar.Z", which this module cannot read, so it is
# rejected along with names that carry no gzip suffix at all.
for name in ('thisisatest.out', 'thisisatest.taz'):
with self.subTest(name=name):
rc, out, err = assert_python_failure('-m', 'gzip', '-d', name)
self.assertEqual(
f"filename doesn't end in .gz or .tgz: {name!r}".encode(),
err.strip())
self.assertEqual(rc, 1)
self.assertEqual(out, b'')

@create_and_remove_directory(TEMPDIR)
def test_decompress_suffix_like_gunzip(self):
# gh-88661: the command line accepts the names gunzip accepts, and
# writes the name gunzip would write.
# Distinct stems: the names differ only in case on some platforms,
# and a case-insensitive filesystem would have them collide.
for name, expected in (('lower.tgz', 'lower.tar'),
('upper.TGZ', 'upper.tar'),
('caps.GZ', 'caps')):
with self.subTest(name=name):
path = os.path.join(TEMPDIR, name)
with gzip.open(path, mode='wb') as fp:
fp.write(self.data)

rc, out, err = assert_python_ok('-m', 'gzip', '-d', path)
self.assertEqual(rc, 0)
self.assertEqual(err, b'')

with open(os.path.join(TEMPDIR, expected), 'rb') as gunziped:
self.assertEqual(gunziped.read(), self.data)

@requires_subprocess()
@create_and_remove_directory(TEMPDIR)
Expand Down
39 changes: 37 additions & 2 deletions Lib/test/test_tarfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -1967,7 +1967,42 @@ def test_missing_fileobj(self):
tar.addfile(tarinfo)


class GzipWriteTest(GzipTest, WriteTest):
class GzipFnameTestBase:
# gh-88661: the FNAME field of the gzip header holds the name that the
# archive is expected to have after decompression. Like gunzip, the
# suffix is matched ignoring case, and ".tgz" becomes ".tar".

def gzip_header_fname(self, path):
with open(path, "rb") as fobj:
header = fobj.read(1024)
self.assertEqual(header[:2], b"\037\213") # gzip magic number
self.assertEqual(header[3], 8) # only the FNAME flag
return header[10:header.index(b"\0", 10)].decode("latin-1")

def test_fname(self):
# Only the suffix is matched ignoring case; the rest of the name
# keeps the case it was given, the way make_ofname() does in gunzip.
for name, expected in (("tmp.tar.gz", "tmp.tar"),
("tmp.TAR.GZ", "tmp.TAR"),
("tmp.tgz", "tmp.tar"),
("tmp.TGZ", "tmp.tar"),
("tmp.tGz", "tmp.tar"),
# .taz means .tar.Z, not gzip.
("tmp.taz", "tmp.taz"),
("tmp.TAZ", "tmp.TAZ"),
("TMP.TGZ", "TMP.tar"),
("TMP.TAZ", "TMP.TAZ"),
("tmp.tar", "tmp.tar")):
with self.subTest(name=name):
path = os.path.join(TEMPDIR, name)
try:
tarfile.open(path, self.mode).close()
self.assertEqual(self.gzip_header_fname(path), expected)
finally:
os_helper.unlink(path)


class GzipWriteTest(GzipTest, GzipFnameTestBase, WriteTest):
pass


Expand Down Expand Up @@ -2034,7 +2069,7 @@ def test_pathlike_name(self):
os_helper.unlink(tmpname)


class GzipStreamWriteTest(GzipTest, StreamWriteTest):
class GzipStreamWriteTest(GzipTest, GzipFnameTestBase, StreamWriteTest):
def test_source_directory_not_leaked(self):
"""
Ensure the source directory is not included in the tar header
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
When writing a gzip header, :mod:`gzip` and :mod:`tarfile` now derive the
``FNAME`` field from the file name like :program:`gunzip` does: the suffix is
matched ignoring case, and ``.tgz`` is replaced with ``.tar`` instead of being
left in place. Previously an archive created as :file:`spam.tgz` recorded
``spam.tgz`` as the name to decompress to. ``python -m gzip -d`` now accepts
the same names, so :file:`spam.tgz` and :file:`spam.GZ` are no longer refused.
Loading