[API Proposal]: APIs to support tar archives #65951

namespace System.Formats.Tar
{
    // Easy to use straightforward archiving and extraction APIs.
    public static class TarFile
    {
        public static void CreateFromDirectory(string sourceDirectoryName, string destinationArchiveFileName, bool includeBaseDirectory);
        public static Task CreateFromDirectoryAsync(string sourceDirectoryName, string destinationArchiveFileName, bool includeBaseDirectory, CancellationToken cancellationToken = default);
        public static void ExtractToDirectory(string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles);
        public static Task ExtractToDirectoryAsync(string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles, CancellationToken cancellationToken = default);
    }

    // Enum representing the entry types that can be detected from V7, Ustar, PAX and GNU.
    public enum TarEntryType : byte
    {
        RegularFileV7 = '\0', // Used exclusively by V7

        // Used  by all formats
        RegularFile = '0',
        HardLink = '1',
        SymbolicLink = '2',
        CharacterDevice = '3',
        BlockDevice = '4',
        Directory = '5',
        Fifo = '6',

        // Exclusively used by PAX
        GlobalExtendedAttributes = 'g',
        ExtendedAttributes = 'x',

        // Exclusively used by GNU
        ContiguousFile = '7',
        DirectoryList = 'D',
        LongLink = 'K',
        LongPath = 'L',
        MultiVolume = 'M',
        RenamedOrSymlinked = 'N',
        SparseFile = 'S',
        TapeVolume = 'T',
    }
    
    // The formats these APIs will be able to read
    public enum TarFormat
    {
        Unknown = 0, // For when an archive that is being read is not recognized
        V7 = 1,
        Ustar = 2,
        Pax = 3,
        Gnu = 4,
    }

    // For traversing entries in an existing tar archive
    public sealed class TarReader : System.IDisposable, System.IAsyncDisposable
    {
        public TarReader(System.IO.Stream archiveStream, bool leaveOpen = false);
        public System.Formats.Tar.TarFormat Format { get; }
        public System.Collections.Generic.IReadOnlyDictionary<string, string>? GlobalExtendedAttributes { get; }
        public void Dispose();
        public ValueTask DisposeAsync();
        public System.Formats.Tar.TarEntry? GetNextEntry(bool copyData = false);
        public ValueTask<TarEntry?> GetNextEntryAsync(bool copyData = false, CancellationToken cancellationToken = default);
    }

    // For creating a tar archive
    public sealed class TarWriter : System.IDisposable, System.IAsyncDisposable
    {
        public TarWriter(System.IO.Stream archiveStream, System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<string, string>>? globalExtendedAttributes = null, bool leaveOpen = false);
        public TarWriter(System.IO.Stream archiveStream, System.Formats.Tar.TarFormat archiveFormat, bool leaveOpen = false);
        public System.Formats.Tar.TarFormat Format { get; }
        public void Dispose();
        public ValueTask DisposeAsync();
        public void WriteEntry(string fileName, string? entryName);
        public Task WriteEntryAsync(string fileName, string? entryName, CancellationToken cancellationToken = default);
        public void WriteEntry(System.Formats.Tar.TarEntry entry);
        public Task WriteEntryAsync(TarEntry entry, CancellationToken cancellationToken = default);
    }

    // Abstract type to represent the header record's metadata fields
    // These fields are found in all the tar formats
    public abstract class TarEntry
    {
        internal TarEntry();
        public int Checksum { get; }
        public System.IO.Stream? DataStream { get; set; }
        public System.Formats.Tar.TarEntryType EntryType { get; }
        public int Gid { get; set; }
        public long Length { get; }
        public string LinkName { get; set; }
        public System.IO.UnixFileMode Mode { get; set; }
        public System.DateTimeOffset MTime { get; set; }
        public string Name { get; set; }
        public int Uid { get; set; }
        public void ExtractToFile(string destinationFileName, bool overwrite);
        public Task ExtractToFileAsync(string destinationFileName, bool overwrite, CancellationToken cancellationToken = default);
        public override string ToString();
    }

    // Allows instancing a V7 tar entry
    public sealed class TarEntryV7 : TarEntry
    {
        public TarEntryV7(System.Formats.Tar.TarEntryType entryType, string entryName);
    }

    // Abstract type that describe the fields proposed by POSIX and inherited by all the subsequent formats
    // GNU also inherits them, but notice that format is not POSIX, so the name for this abstract type can be different
    public abstract class TarEntryPosix : TarEntry
    {
        internal TarEntryPosix();
        public int DeviceMajor { get; set; }
        public int DeviceMinor { get; set; }
        public string GName { get; set; }
        public string UName { get; set; }
        public override string ToString();
    }

    // Allows instancing a Ustar tar entry, which is the first POSIX format
    public sealed class TarEntryUstar : TarEntryPosix
    {
        public TarEntryUstar(System.Formats.Tar.TarEntryType entryType, string entryName);
    }

    // Allows instancing a PAX tar entry
    // Contains a dictionary that exposes the extended attributes found in the extra metadata entry the Pax format defines
    public sealed class TarEntryPax : TarEntryPosix
    {
        public TarEntryPax(System.Formats.Tar.TarEntryType entryType, string entryName, System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<string, string>>? extendedAttributes);
        public System.Collections.Generic.IReadOnlyDictionary<string, string> ExtendedAttributes { get; }
    }

    // Allows instancing a GNU tar entry
    // Contains additional metadata fields found in GNU
    public sealed class TarEntryGnu : TarEntryPosix
    {
        public TarEntryGnu(System.Formats.Tar.TarEntryType entryType, string entryName);
        public System.DateTimeOffset ATime { get; set; }
        public System.DateTimeOffset CTime { get; set; }
    }
}

namespace System.IO
{
    [System.FlagsAttribute]
    public enum UnixFileMode
    {
        None = 0,
        OtherExecute = 1,
        OtherWrite = 2,
        OtherRead = 4,
        GroupExecute = 8,
        GroupWrite = 16,
        GroupRead = 32,
        UserExecute = 64,
        UserWrite = 128,
        UserRead = 256,
        StickyBit = 512,
        GroupSpecial = 1024,
        UserSpecial = 2048,
    }
}

// Generates a tar archive where all the entry paths are prefixed by the root directory 'SourceDirectory'
TarFile.CreateFromDirectory(sourceDirectoryName: "D:/SourceDirectory/", destinationArchiveFileName: "D:/destination.tar", includeBaseDirectory: true);

// Extracts the contents of a tar archive into the specified directory, but avoiding overwriting anything found inside
TarFile.ExtractToDirectory(sourceArchiveFileName: "D:/destination.tar", destinationDirectoryName: "D:/DestinationDirectory/", overwriteFiles: false);

FileStream archiveStream = File.Open("D:/archive.tar", FileMode.Open, FileAccess.Read);

using TarReader reader = new TarReader(archiveStream, leaveOpen: true);

Console.WriteLine($"Format: {reader.Format}");

if (reader.GlobalExtendedAttributes != null)
{
    Console.WriteLine("Format is PAX");
}

TarEntry? entry;
while ((entry = reader.GetNextEntry()) != null)
{
    Console.WriteLine($"Entry name: {entry.Name}, entry type: {entry.EntryType}");
    entry.ExtractToFile(destinationFileName: Path.Join("D:/MyExtractionFolder/", entry.Name), overwrite: false);
}

public void ReadTarFromNetwork(NetworkStream archiveStream) // This stream is not seekable
{
    using TarReader reader = new TarReader(archiveStream);

    while ((entry = reader.GetNextEntry(copyData: false)) != null) // Not copying the data means it needs to be read now, before advancing the stream position
    {
        if (entry.EntryType is TarEntryType.RegularFile)
        {
            // This needs to be done now because the position pointer will not be able to seek back later
            entry.ExtractToFile(destinationFileName: Path.Join("D:/MyExtractionFolder/", entry.Name), overwrite: false);

            DoSomethingWithTheData(entry.DataStream); // This won't be possible since the data stream position pointer is at the end of the stream
        }
    }
}

public void ReadTarFromNetwork(NetworkStream archiveStream) // This stream is not seekable
{
    List<TarEntry> entries = new List<TarEntry>();

    using TarReader reader = new TarReader(archiveStream);

    while ((entry = reader.GetNextEntry(copyData: true)) != null) // Copy the data internally for later usage
    {
        entries.Add(entry);
    } // Stream position is now located at the end of the stream

    foreach (TarEntry entry in entries)
    {
        if (entry.EntryType is TarEntryType.RegularFile)
        {
            // This is possible because the data was saved internally
            entry.ExtractToFile(destinationFileName: Path.Join("D:/MyExtractionFolder/", entry.Name), overwrite: false);

            // We can also inspect the data stream now
            entry.DataStream.Seek(0, SeekOrigin.Begin);
            DoSomethingWithTheData(entry.DataStream);
        }
    }
    
}

FileStream archiveStream = File.Create("D:/archive.tar");

using TarWriter writerV7 = new TarWriter(archiveStream, TarFormat.V7);

using TarWriter writerUstar = new TarWriter(archiveStream, TarFormat.Ustar);

using TarWriter writerPax1 = new TarWriter(archiveStream, TarFormat.Pax); // No Global Extended Attributes entry

Dictionary<string, string> gea = new Dictionary<string, string>();
gea.Add("something", "global");
using TarWriter writerPaxGEA = new TarWriter(archiveStream, globalExtendedAttributes: dictionary); // Note there's no need to indicate the format, it's assumed

using TarWriter writerGnu = new TarWriter(archiveStream, TarFormat.Gnu);

// EntryType: Directory
writer.WriteEntry(fileName: "D:/IAmADirectory/", entryName: "IAmADirectory");

// EntryType: RegularFile (or if V7: OldRegularFile)
writer.WriteEntry(fileName: "D:/file.txt", entryName: "file.txt");

// In Unix, if the writer was opened in Ustar, Pax or Gnu, the user can also add fifo, block device and character device files to the archive
writer.WriteEntry(fileName: "/home/carlos/myfifo", entryName: "myfifo"); // EntryType: Fifo
writer.WriteEntry(fileName: "/home/carlos/myblockdevice", entryName: "myblockdevice"); // EntryType: BlockDevice
writer.WriteEntry(fileName: "/home/carlos/mycharacterdevice", entryName: "mychardevice"); // EntryType: CharDevice

TarEntryV7 entry = new TarEntryV7(entryType: TarEntryType.OldRegularFile, entryName: "file.txt");
using (FileStream dataStream = File.Open("D:/file.txt", FileMode.Open, FileAccess.Read))
{
  entry.DataStream = dataStream;
  entry.Gid = 5;
  entry.Uid = 7;
  writerV7.WriteEntry(entry);
} // The user created the data stream externally, so they need to dispose it

TarEntryUstar entry = new TarEntryUstar(entryType: TarEntryType.RegularFile, entryName: "file.txt");
entry.DataStream = File.Open("D:/file.txt", FileMode.Open, FileAccess.Read);
entry.Mode = UnixFileMode.UserRead | UnixFileMode.GroupRead | UnixFileMode.OtherRead;
entry.UName = "carlos";
entry.GName = "dotnet";
writerUstar.WriteEntry(entry);

TarEntryPax entry = new TarEntryPax(entryType: TarEntryType.Directory, entryName: "directory", extendedAttributes: null); // No extended attributes, but the metadata header is created anyway
writerPax.WriteEntry(entry);

Dictionary<string, string> ea = new DictionaryString<ea, ea>();
ea.Add("atime", $"{DateTimeOffset.Now}");
ea.Add("ctime", $"{DateTimeOffset.Now}");
TarEntryPax entryWithEA = new TarEntryPax(entryType: TarEntryType.SymbolicLink, entryName: "symlink", extendedAttributes: ea);
entryWithEA.LinkName = "this/is/a/link/path";
writer.WriteEntry(entryWithEA);

TarEntryGnu entry = new TarEntryGnu(entryType: TarEntryType.CharacterDevice, entryName: "chardevice");
entry.DeviceMajor = 444;
entry.DeviceMinor = 555;
entry.ATime = DateTimeOffset.Now;
entry.CTime = DateTimeOffset.Now;
writerGnu.WriteEntry(entry);

using TarReader reader = new TarReader(originStream); // The detected format of this archive should not matter
using TarWriter writer = new TarWriter(destinationStream, TarFormat.Pax);

TarEntry? entry;
while ((entry = reader.GetNextEntry(copyData: true)) != null)
{
    writer.WriteEntry(entry); // Entries should be saved in PAX format, reading as much as possible from the passed entry in a different format
}

MemoryStream archiveStream = new MemoryStream();
using (TarWriter writer = new TarWriter(archiveStream, TarFormat.Pax, leaveOpen: true)) // Do not close stream on dispose
{
    TarEntryPax entry = new TarEntryPax(entryType: TarEntryType.RegularFile, entryName: "file.txt");
    // ... configure the entry ...
    writer.WriteEntry(entry);
} // Dispose triggers writing the empty records at the end of the archive

using FileStream compressedFileStream = File.Create("file.tar.gz");
using GZipStream compressor = new GZipStream(compressedFileStream, CompressionMode.Compress);
archiveStream.CopyTo(compressor); // After disposing these two, the tar.gz will be commited